Mailman 3 August 2021 - Python-checkins

bpo-42064: Offset arguments for PyObject_Vectorcall in the _sqlite module (GH-27931)
by encukou Aug. 31, 2021

Aug. 31, 2021

https://github.com/python/cpython/commit/01dea5f12b31862999217c091399a318f2… commit: 01dea5f12b31862999217c091399a318f23b460a branch: main author: Petr Viktorin <encukou(a)gmail.com> committer: encukou <encukou(a)gmail.com> date: 2021-08-31T14:34:44+02:00 summary: bpo-42064: Offset arguments for PyObject_Vectorcall in the _sqlite module (GH-27931) This allows e.g. methods to be called efficiently by providing space for a "self" argument; see PY_VECTORCALL_ARGUMENTS_OFFSET docs. files: M Modules/_sqlite/connection.c M Modules/_sqlite/cursor.c diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 9ff5fbaae7852..864877c0c3426 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -59,19 +59,21 @@ static void _pysqlite_drop_unused_cursor_references(pysqlite_Connection* self); static PyObject * new_statement_cache(pysqlite_Connection *self, int maxsize) { - PyObject *args[] = { PyLong_FromLong(maxsize), }; - if (args[0] == NULL) { + PyObject *args[] = { NULL, PyLong_FromLong(maxsize), }; + if (args[1] == NULL) { return NULL; } PyObject *lru_cache = self->state->lru_cache; - PyObject *inner = PyObject_Vectorcall(lru_cache, args, 1, NULL); - Py_DECREF(args[0]); + size_t nargsf = 1 | PY_VECTORCALL_ARGUMENTS_OFFSET; + PyObject *inner = PyObject_Vectorcall(lru_cache, args + 1, nargsf, NULL); + Py_DECREF(args[1]); if (inner == NULL) { return NULL; } - args[0] = (PyObject *)self; // Borrowed ref. - PyObject *res = PyObject_Vectorcall(inner, args, 1, NULL); + args[1] = (PyObject *)self; // Borrowed ref. + nargsf = 1 | PY_VECTORCALL_ARGUMENTS_OFFSET; + PyObject *res = PyObject_Vectorcall(inner, args + 1, nargsf, NULL); Py_DECREF(inner); return res; } @@ -1474,8 +1476,9 @@ pysqlite_collation_callback( callback_context *ctx = (callback_context *)context; assert(ctx != NULL); - PyObject *args[] = { string1, string2 }; // Borrowed refs. - retval = PyObject_Vectorcall(ctx->callable, args, 2, NULL); + PyObject *args[] = { NULL, string1, string2 }; // Borrowed refs. + size_t nargsf = 2 | PY_VECTORCALL_ARGUMENTS_OFFSET; + retval = PyObject_Vectorcall(ctx->callable, args + 1, nargsf, NULL); if (retval == NULL) { /* execution failed */ goto finally; diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index 8b830ec767b5d..06ce385cca3a4 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -462,9 +462,10 @@ begin_transaction(pysqlite_Connection *self) static PyObject * get_statement_from_cache(pysqlite_Cursor *self, PyObject *operation) { - PyObject *args[] = { operation, }; + PyObject *args[] = { NULL, operation, }; // Borrowed ref. PyObject *cache = self->connection->statement_cache; - return PyObject_Vectorcall(cache, args, 1, NULL); + size_t nargsf = 1 | PY_VECTORCALL_ARGUMENTS_OFFSET; + return PyObject_Vectorcall(cache, args + 1, nargsf, NULL); } static PyObject *

1 0

bpo-44991: Make GIL handling more explicit in `sqlite3` callbacks (GH-27934)
by miss-islington Aug. 31, 2021

Aug. 31, 2021

https://github.com/python/cpython/commit/001ef4600f5ab2e1d7825ddbc0f253377c… commit: 001ef4600f5ab2e1d7825ddbc0f253377c234d7e branch: main author: Erlend Egeberg Aasland <erlend.aasland(a)innova.no> committer: miss-islington <31488909+miss-islington(a)users.noreply.github.com> date: 2021-08-31T05:18:43-07:00 summary: bpo-44991: Make GIL handling more explicit in `sqlite3` callbacks (GH-27934) - acquire the GIL at the very start[1] - release the GIL at the very end [1] The trace callback performs a sanity check before acquiring the GIL Automerge-Triggered-By: GH:encukou files: M Modules/_sqlite/connection.c diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 19d30d24d7f2e7..9ff5fbaae78529 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -626,14 +626,12 @@ set_sqlite_error(sqlite3_context *context, const char *msg) static void _pysqlite_func_callback(sqlite3_context *context, int argc, sqlite3_value **argv) { + PyGILState_STATE threadstate = PyGILState_Ensure(); + PyObject* args; PyObject* py_retval = NULL; int ok; - PyGILState_STATE threadstate; - - threadstate = PyGILState_Ensure(); - args = _pysqlite_build_py_params(context, argc, argv); if (args) { callback_context *ctx = (callback_context *)sqlite3_user_data(context); @@ -656,15 +654,13 @@ _pysqlite_func_callback(sqlite3_context *context, int argc, sqlite3_value **argv static void _pysqlite_step_callback(sqlite3_context *context, int argc, sqlite3_value** params) { + PyGILState_STATE threadstate = PyGILState_Ensure(); + PyObject* args; PyObject* function_result = NULL; PyObject** aggregate_instance; PyObject* stepmethod = NULL; - PyGILState_STATE threadstate; - - threadstate = PyGILState_Ensure(); - aggregate_instance = (PyObject**)sqlite3_aggregate_context(context, sizeof(PyObject*)); if (*aggregate_instance == NULL) { @@ -706,16 +702,14 @@ static void _pysqlite_step_callback(sqlite3_context *context, int argc, sqlite3_ static void _pysqlite_final_callback(sqlite3_context *context) { + PyGILState_STATE threadstate = PyGILState_Ensure(); + PyObject* function_result; PyObject** aggregate_instance; _Py_IDENTIFIER(finalize); int ok; PyObject *exception, *value, *tb; - PyGILState_STATE threadstate; - - threadstate = PyGILState_Ensure(); - aggregate_instance = (PyObject**)sqlite3_aggregate_context(context, 0); if (aggregate_instance == NULL) { /* No rows matched the query; the step handler was never called. */ @@ -787,34 +781,34 @@ static void _pysqlite_drop_unused_cursor_references(pysqlite_Connection* self) static callback_context * create_callback_context(pysqlite_state *state, PyObject *callable) { - PyGILState_STATE gstate = PyGILState_Ensure(); callback_context *ctx = PyMem_Malloc(sizeof(callback_context)); if (ctx != NULL) { ctx->callable = Py_NewRef(callable); ctx->state = state; } - PyGILState_Release(gstate); return ctx; } static void free_callback_context(callback_context *ctx) +{ + assert(ctx != NULL); + Py_DECREF(ctx->callable); + PyMem_Free(ctx); +} + +static void +_destructor(void *ctx) { if (ctx != NULL) { // This function may be called without the GIL held, so we need to // ensure that we destroy 'ctx' with the GIL held. PyGILState_STATE gstate = PyGILState_Ensure(); - Py_DECREF(ctx->callable); - PyMem_Free(ctx); + free_callback_context((callback_context *)ctx); PyGILState_Release(gstate); } } -static void _destructor(void* args) -{ - free_callback_context((callback_context *)args); -} - /*[clinic input] _sqlite3.Connection.create_function as pysqlite_connection_create_function @@ -914,11 +908,10 @@ pysqlite_connection_create_aggregate_impl(pysqlite_Connection *self, static int _authorizer_callback(void* user_arg, int action, const char* arg1, const char* arg2 , const char* dbname, const char* access_attempt_source) { + PyGILState_STATE gilstate = PyGILState_Ensure(); + PyObject *ret; int rc; - PyGILState_STATE gilstate; - - gilstate = PyGILState_Ensure(); ret = PyObject_CallFunction((PyObject*)user_arg, "issss", action, arg1, arg2, dbname, access_attempt_source); @@ -959,11 +952,10 @@ static int _authorizer_callback(void* user_arg, int action, const char* arg1, co static int _progress_handler(void* user_arg) { + PyGILState_STATE gilstate = PyGILState_Ensure(); + int rc; PyObject *ret; - PyGILState_STATE gilstate; - - gilstate = PyGILState_Ensure(); ret = _PyObject_CallNoArg((PyObject*)user_arg); if (!ret) { @@ -1000,18 +992,16 @@ static int _trace_callback(unsigned int type, void* user_arg, void* prepared_sta static void _trace_callback(void* user_arg, const char* statement_string) #endif { - PyObject *py_statement = NULL; - PyObject *ret = NULL; - - PyGILState_STATE gilstate; - #ifdef HAVE_TRACE_V2 if (type != SQLITE_TRACE_STMT) { return 0; } #endif - gilstate = PyGILState_Ensure(); + PyGILState_STATE gilstate = PyGILState_Ensure(); + + PyObject *py_statement = NULL; + PyObject *ret = NULL; py_statement = PyUnicode_DecodeUTF8(statement_string, strlen(statement_string), "replace"); if (py_statement) { @@ -1461,14 +1451,16 @@ pysqlite_collation_callback( int text1_length, const void* text1_data, int text2_length, const void* text2_data) { + PyGILState_STATE gilstate = PyGILState_Ensure(); + PyObject* string1 = 0; PyObject* string2 = 0; - PyGILState_STATE gilstate; PyObject* retval = NULL; long longval; int result = 0; - gilstate = PyGILState_Ensure(); + /* This callback may be executed multiple times per sqlite3_step(). Bail if + * the previous call failed */ if (PyErr_Occurred()) { goto finally; }

1 0

bpo-44925: [docs] Fix confusing deprecation notice for typing.IO (GH-28004)
by Fidget-Spinner Aug. 31, 2021

Aug. 31, 2021

https://github.com/python/cpython/commit/edae42f99f8153b92ccf365dbd1c2fa954… commit: edae42f99f8153b92ccf365dbd1c2fa954f913b4 branch: main author: DonnaDia <37962843+DonnaDia(a)users.noreply.github.com> committer: Fidget-Spinner <28750310+Fidget-Spinner(a)users.noreply.github.com> date: 2021-08-31T17:44:27+08:00 summary: bpo-44925: [docs] Fix confusing deprecation notice for typing.IO (GH-28004) files: M Doc/library/typing.rst diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 53cf5429b54c97..13760c19214ee7 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -1507,8 +1507,8 @@ Other concrete types :func:`open`. .. deprecated-removed:: 3.8 3.12 - These types are also in the ``typing.io`` namespace, which was - never supported by type checkers and will be removed. + The ``typing.io`` namespace is deprecated and will be removed. + These types should be directly imported from ``typing`` instead. .. class:: Pattern Match @@ -1521,8 +1521,8 @@ Other concrete types ``Match[bytes]``. .. deprecated-removed:: 3.8 3.12 - These types are also in the ``typing.re`` namespace, which was - never supported by type checkers and will be removed. + The ``typing.re`` namespace is deprecated and will be removed. + These types should be directly imported from ``typing`` instead. .. deprecated:: 3.9 Classes ``Pattern`` and ``Match`` from :mod:`re` now support ``[]``.

1 0

[3.6] bpo-44394: Update libexpat copy to 2.4.1 (GH-26945) (GH-28042) (GH-28080)
by ned-deily Aug. 30, 2021

Aug. 30, 2021

https://github.com/python/cpython/commit/910886a6448e4bf1edf49eeace4aa240b6… commit: 910886a6448e4bf1edf49eeace4aa240b6403772 branch: 3.6 author: Ned Deily <nad(a)python.org> committer: ned-deily <nad(a)python.org> date: 2021-08-31T02:35:31-04:00 summary: [3.6] bpo-44394: Update libexpat copy to 2.4.1 (GH-26945) (GH-28042) (GH-28080) Update the vendored copy of libexpat to 2.4.1 (from 2.2.8) to get the fix for the CVE-2013-0340 "Billion Laughs" vulnerability. This copy is most used on Windows and macOS. Co-authored-by: Victor Stinner <vstinner(a)python.org> Co-authored-by: Łukasz Langa <lukasz(a)langa.pl>. (cherry picked from commit 3fc5d84046ddbd66abac5b598956ea34605a4e5d) files: A Misc/NEWS.d/next/Security/2021-06-29-02-45-53.bpo-44394.A220N1.rst M Doc/library/xml.rst M Doc/whatsnew/3.6.rst M Modules/expat/COPYING M Modules/expat/ascii.h M Modules/expat/asciitab.h M Modules/expat/expat.h M Modules/expat/expat_external.h M Modules/expat/iasciitab.h M Modules/expat/internal.h M Modules/expat/latin1tab.h M Modules/expat/nametab.h M Modules/expat/siphash.h M Modules/expat/utf8tab.h M Modules/expat/winconfig.h M Modules/expat/xmlparse.c M Modules/expat/xmlrole.c M Modules/expat/xmlrole.h M Modules/expat/xmltok.c M Modules/expat/xmltok.h M Modules/expat/xmltok_impl.c M Modules/expat/xmltok_impl.h M Modules/expat/xmltok_ns.c diff --git a/Doc/library/xml.rst b/Doc/library/xml.rst index 9b8ba6b17c8531..ac92a0d3a013ba 100644 --- a/Doc/library/xml.rst +++ b/Doc/library/xml.rst @@ -60,23 +60,27 @@ circumvent firewalls. The following table gives an overview of the known attacks and whether the various modules are vulnerable to them. -========================= ============== =============== ============== ============== ============== -kind sax etree minidom pulldom xmlrpc -========================= ============== =============== ============== ============== ============== -billion laughs **Vulnerable** **Vulnerable** **Vulnerable** **Vulnerable** **Vulnerable** -quadratic blowup **Vulnerable** **Vulnerable** **Vulnerable** **Vulnerable** **Vulnerable** -external entity expansion Safe (4) Safe (1) Safe (2) Safe (4) Safe (3) -`DTD`_ retrieval Safe (4) Safe Safe Safe (4) Safe -decompression bomb Safe Safe Safe Safe **Vulnerable** -========================= ============== =============== ============== ============== ============== - -1. :mod:`xml.etree.ElementTree` doesn't expand external entities and raises a +========================= ================== ================== ================== ================== ================== +kind sax etree minidom pulldom xmlrpc +========================= ================== ================== ================== ================== ================== +billion laughs **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) +quadratic blowup **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) +external entity expansion Safe (5) Safe (2) Safe (3) Safe (5) Safe (4) +`DTD`_ retrieval Safe (5) Safe Safe Safe (5) Safe +decompression bomb Safe Safe Safe Safe **Vulnerable** +========================= ================== ================== ================== ================== ================== + +1. Expat 2.4.1 and newer is not vulnerable to the "billion laughs" and + "quadratic blowup" vulnerabilities. Items still listed as vulnerable due to + potential reliance on system-provided libraries. Check + :data:`pyexpat.EXPAT_VERSION`. +2. :mod:`xml.etree.ElementTree` doesn't expand external entities and raises a :exc:`ParserError` when an entity occurs. -2. :mod:`xml.dom.minidom` doesn't expand external entities and simply returns +3. :mod:`xml.dom.minidom` doesn't expand external entities and simply returns the unexpanded entity verbatim. -3. :mod:`xmlrpclib` doesn't expand external entities and omits them. -4. Since Python 3.8.0, external general entities are no longer processed by - default since Python. +4. :mod:`xmlrpclib` doesn't expand external entities and omits them. +5. Since Python 3.6.7, external general entities are no longer processed by + default. billion laughs / exponential entity expansion diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index c14e790935a208..5f8f478eb37945 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -2181,7 +2181,7 @@ Changes in the Python API * The functions in the :mod:`compileall` module now return booleans instead of ``1`` or ``0`` to represent success or failure, respectively. Thanks to - booleans being a subclass of integers, this should only be an issue if you7 + booleans being a subclass of integers, this should only be an issue if you were doing identity checks for ``1`` or ``0``. See :issue:`25768`. * Reading the :attr:`~urllib.parse.SplitResult.port` attribute of diff --git a/Misc/NEWS.d/next/Security/2021-06-29-02-45-53.bpo-44394.A220N1.rst b/Misc/NEWS.d/next/Security/2021-06-29-02-45-53.bpo-44394.A220N1.rst new file mode 100644 index 00000000000000..e32563d2535c7e --- /dev/null +++ b/Misc/NEWS.d/next/Security/2021-06-29-02-45-53.bpo-44394.A220N1.rst @@ -0,0 +1,3 @@ +Update the vendored copy of libexpat to 2.4.1 (from 2.2.8) to get the fix +for the CVE-2013-0340 "Billion Laughs" vulnerability. This copy is most used +on Windows and macOS. diff --git a/Modules/expat/COPYING b/Modules/expat/COPYING index 8d288f0f28fddd..3c0142e71c8d4f 100644 --- a/Modules/expat/COPYING +++ b/Modules/expat/COPYING @@ -1,5 +1,5 @@ Copyright (c) 1998-2000 Thai Open Source Software Center Ltd and Clark Cooper -Copyright (c) 2001-2017 Expat maintainers +Copyright (c) 2001-2019 Expat maintainers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/Modules/expat/ascii.h b/Modules/expat/ascii.h index c3587e57332bff..1f594d2e54b4d2 100644 --- a/Modules/expat/ascii.h +++ b/Modules/expat/ascii.h @@ -6,8 +6,11 @@ \___/_/\_\ .__/ \__,_|\__| |_| XML parser - Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 1999-2000 Thai Open Source Software Center Ltd + Copyright (c) 2000 Clark Cooper <coopercc(a)users.sourceforge.net> + Copyright (c) 2002 Fred L. Drake, Jr. <fdrake(a)users.sourceforge.net> + Copyright (c) 2007 Karl Waclawek <karl(a)waclawek.net> + Copyright (c) 2017 Sebastian Pipping <sebastian(a)pipping.org> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining diff --git a/Modules/expat/asciitab.h b/Modules/expat/asciitab.h index 63b1d1b4482efa..af766fb24785ea 100644 --- a/Modules/expat/asciitab.h +++ b/Modules/expat/asciitab.h @@ -7,7 +7,9 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper <coopercc(a)users.sourceforge.net> + Copyright (c) 2002 Fred L. Drake, Jr. <fdrake(a)users.sourceforge.net> + Copyright (c) 2017 Sebastian Pipping <sebastian(a)pipping.org> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index 6c8eb1fda4a30c..b7d6d354801b3d 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -7,7 +7,14 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper <coopercc(a)users.sourceforge.net> + Copyright (c) 2000-2005 Fred L. Drake, Jr. <fdrake(a)users.sourceforge.net> + Copyright (c) 2001-2002 Greg Stein <gstein(a)users.sourceforge.net> + Copyright (c) 2002-2016 Karl Waclawek <karl(a)waclawek.net> + Copyright (c) 2016-2021 Sebastian Pipping <sebastian(a)pipping.org> + Copyright (c) 2016 Cristian Rodríguez <crrodriguez(a)opensuse.org> + Copyright (c) 2016 Thomas Beutlich <tc(a)tbeu.de> + Copyright (c) 2017 Rhodri James <rhodri(a)wildebeest.org.uk> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -115,7 +122,11 @@ enum XML_Error { XML_ERROR_RESERVED_PREFIX_XMLNS, XML_ERROR_RESERVED_NAMESPACE_URI, /* Added in 2.2.1. */ - XML_ERROR_INVALID_ARGUMENT + XML_ERROR_INVALID_ARGUMENT, + /* Added in 2.3.0. */ + XML_ERROR_NO_BUFFER, + /* Added in 2.4.0. */ + XML_ERROR_AMPLIFICATION_LIMIT_BREACH }; enum XML_Content_Type { @@ -318,7 +329,7 @@ typedef void(XMLCALL *XML_EndDoctypeDeclHandler)(void *userData); For internal entities (<!ENTITY foo "bar">), value will be non-NULL and systemId, publicID, and notationName will be NULL. - The value string is NOT nul-terminated; the length is provided in + The value string is NOT null-terminated; the length is provided in the value_length argument. Since it is legal to have zero-length values, do not use this argument to test for internal entities. @@ -513,7 +524,7 @@ typedef struct { Otherwise it must return XML_STATUS_ERROR. If info does not describe a suitable encoding, then the parser will - return an XML_UNKNOWN_ENCODING error. + return an XML_ERROR_UNKNOWN_ENCODING error. */ typedef int(XMLCALL *XML_UnknownEncodingHandler)(void *encodingHandlerData, const XML_Char *name, @@ -707,7 +718,7 @@ XML_GetBase(XML_Parser parser); /* Returns the number of the attribute/value pairs passed in last call to the XML_StartElementHandler that were specified in the start-tag rather than defaulted. Each attribute/value pair counts as 2; thus - this correspondds to an index into the atts array passed to the + this corresponds to an index into the atts array passed to the XML_StartElementHandler. Returns -1 if parser == NULL. */ XMLPARSEAPI(int) @@ -716,7 +727,7 @@ XML_GetSpecifiedAttributeCount(XML_Parser parser); /* Returns the index of the ID attribute passed in the last call to XML_StartElementHandler, or -1 if there is no ID attribute or parser == NULL. Each attribute/value pair counts as 2; thus this - correspondds to an index into the atts array passed to the + corresponds to an index into the atts array passed to the XML_StartElementHandler. */ XMLPARSEAPI(int) @@ -997,7 +1008,10 @@ enum XML_FeatureEnum { XML_FEATURE_SIZEOF_XML_LCHAR, XML_FEATURE_NS, XML_FEATURE_LARGE_SIZE, - XML_FEATURE_ATTR_INFO + XML_FEATURE_ATTR_INFO, + /* Added in Expat 2.4.0. */ + XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT, + XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT /* Additional features must be added to the end of this enum. */ }; @@ -1010,12 +1024,24 @@ typedef struct { XMLPARSEAPI(const XML_Feature *) XML_GetFeatureList(void); +#ifdef XML_DTD +/* Added in Expat 2.4.0. */ +XMLPARSEAPI(XML_Bool) +XML_SetBillionLaughsAttackProtectionMaximumAmplification( + XML_Parser parser, float maximumAmplificationFactor); + +/* Added in Expat 2.4.0. */ +XMLPARSEAPI(XML_Bool) +XML_SetBillionLaughsAttackProtectionActivationThreshold( + XML_Parser parser, unsigned long long activationThresholdBytes); +#endif + /* Expat follows the semantic versioning convention. See http://semver.org. */ #define XML_MAJOR_VERSION 2 -#define XML_MINOR_VERSION 2 -#define XML_MICRO_VERSION 8 +#define XML_MINOR_VERSION 4 +#define XML_MICRO_VERSION 1 #ifdef __cplusplus } diff --git a/Modules/expat/expat_external.h b/Modules/expat/expat_external.h index f2b75dda8e2798..12c560e14716ff 100644 --- a/Modules/expat/expat_external.h +++ b/Modules/expat/expat_external.h @@ -7,7 +7,14 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper <coopercc(a)users.sourceforge.net> + Copyright (c) 2000-2004 Fred L. Drake, Jr. <fdrake(a)users.sourceforge.net> + Copyright (c) 2001-2002 Greg Stein <gstein(a)users.sourceforge.net> + Copyright (c) 2002-2006 Karl Waclawek <karl(a)waclawek.net> + Copyright (c) 2016 Cristian Rodríguez <crrodriguez(a)opensuse.org> + Copyright (c) 2016-2019 Sebastian Pipping <sebastian(a)pipping.org> + Copyright (c) 2017 Rhodri James <rhodri(a)wildebeest.org.uk> + Copyright (c) 2018 Yury Gribov <tetra2005(a)gmail.com> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining diff --git a/Modules/expat/iasciitab.h b/Modules/expat/iasciitab.h index ea97cfcf678e06..5d8646f2a318b8 100644 --- a/Modules/expat/iasciitab.h +++ b/Modules/expat/iasciitab.h @@ -7,7 +7,9 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper <coopercc(a)users.sourceforge.net> + Copyright (c) 2002 Fred L. Drake, Jr. <fdrake(a)users.sourceforge.net> + Copyright (c) 2017 Sebastian Pipping <sebastian(a)pipping.org> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining diff --git a/Modules/expat/internal.h b/Modules/expat/internal.h index 60913dab762f8f..444eba0fb03170 100644 --- a/Modules/expat/internal.h +++ b/Modules/expat/internal.h @@ -25,8 +25,12 @@ \___/_/\_\ .__/ \__,_|\__| |_| XML parser - Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2002-2003 Fred L. Drake, Jr. <fdrake(a)users.sourceforge.net> + Copyright (c) 2002-2006 Karl Waclawek <karl(a)waclawek.net> + Copyright (c) 2003 Greg Stein <gstein(a)users.sourceforge.net> + Copyright (c) 2016-2021 Sebastian Pipping <sebastian(a)pipping.org> + Copyright (c) 2018 Yury Gribov <tetra2005(a)gmail.com> + Copyright (c) 2019 David Loffredo <loffredo(a)steptools.com> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -101,22 +105,58 @@ # endif #endif +#include <limits.h> // ULONG_MAX + +#if defined(_WIN32) && ! defined(__USE_MINGW_ANSI_STDIO) +# define EXPAT_FMT_ULL(midpart) "%" midpart "I64u" +# if defined(_WIN64) // Note: modifiers "td" and "zu" do not work for MinGW +# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "I64d" +# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "I64u" +# else +# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "d" +# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "u" +# endif +#else +# define EXPAT_FMT_ULL(midpart) "%" midpart "llu" +# if ! defined(ULONG_MAX) +# error Compiler did not define ULONG_MAX for us +# elif ULONG_MAX == 18446744073709551615u // 2^64-1 +# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "ld" +# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "lu" +# else +# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "d" +# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "u" +# endif +#endif + #ifndef UNUSED_P # define UNUSED_P(p) (void)p #endif +/* NOTE BEGIN If you ever patch these defaults to greater values + for non-attack XML payload in your environment, + please file a bug report with libexpat. Thank you! +*/ +#define EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT \ + 100.0f +#define EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT \ + 8388608 // 8 MiB, 2^23 +/* NOTE END */ + +#include "expat.h" // so we can use type XML_Parser below + #ifdef __cplusplus extern "C" { #endif -#ifdef XML_ENABLE_VISIBILITY -# if XML_ENABLE_VISIBILITY -__attribute__((visibility("default"))) -# endif +void _INTERNAL_trim_to_complete_utf8_characters(const char *from, + const char **fromLimRef); + +#if defined(XML_DTD) +unsigned long long testingAccountingGetCountBytesDirect(XML_Parser parser); +unsigned long long testingAccountingGetCountBytesIndirect(XML_Parser parser); +const char *unsignedCharToPrintable(unsigned char c); #endif -void -_INTERNAL_trim_to_complete_utf8_characters(const char *from, - const char **fromLimRef); #ifdef __cplusplus } diff --git a/Modules/expat/latin1tab.h b/Modules/expat/latin1tab.h index 6f916041355a86..b681d278af6569 100644 --- a/Modules/expat/latin1tab.h +++ b/Modules/expat/latin1tab.h @@ -7,7 +7,9 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper <coopercc(a)users.sourceforge.net> + Copyright (c) 2002 Fred L. Drake, Jr. <fdrake(a)users.sourceforge.net> + Copyright (c) 2017 Sebastian Pipping <sebastian(a)pipping.org> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining diff --git a/Modules/expat/nametab.h b/Modules/expat/nametab.h index 3681df348eebd6..63485446b96727 100644 --- a/Modules/expat/nametab.h +++ b/Modules/expat/nametab.h @@ -6,8 +6,8 @@ \___/_/\_\ .__/ \__,_|\__| |_| XML parser - Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper <coopercc(a)users.sourceforge.net> + Copyright (c) 2017 Sebastian Pipping <sebastian(a)pipping.org> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining diff --git a/Modules/expat/siphash.h b/Modules/expat/siphash.h index bfee65a332f1bf..e5406d7ee9eb54 100644 --- a/Modules/expat/siphash.h +++ b/Modules/expat/siphash.h @@ -11,6 +11,9 @@ * -------------------------------------------------------------------------- * HISTORY: * + * 2020-10-03 (Sebastian Pipping) + * - Drop support for Visual Studio 9.0/2008 and earlier + * * 2019-08-03 (Sebastian Pipping) * - Mark part of sip24_valid as to be excluded from clang-format * - Re-format code using clang-format 9 @@ -96,15 +99,7 @@ #define SIPHASH_H #include <stddef.h> /* size_t */ - -#if defined(_WIN32) && defined(_MSC_VER) && (_MSC_VER < 1600) -/* For vs2003/7.1 up to vs2008/9.0; _MSC_VER 1600 is vs2010/10.0 */ -typedef unsigned __int8 uint8_t; -typedef unsigned __int32 uint32_t; -typedef unsigned __int64 uint64_t; -#else -# include <stdint.h> /* uint64_t uint32_t uint8_t */ -#endif +#include <stdint.h> /* uint64_t uint32_t uint8_t */ /* * Workaround to not require a C++11 compiler for using ULL suffix diff --git a/Modules/expat/utf8tab.h b/Modules/expat/utf8tab.h index a22986acbb9526..88efcf91cc16a6 100644 --- a/Modules/expat/utf8tab.h +++ b/Modules/expat/utf8tab.h @@ -7,7 +7,9 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper <coopercc(a)users.sourceforge.net> + Copyright (c) 2002 Fred L. Drake, Jr. <fdrake(a)users.sourceforge.net> + Copyright (c) 2017 Sebastian Pipping <sebastian(a)pipping.org> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining diff --git a/Modules/expat/winconfig.h b/Modules/expat/winconfig.h index 562a4a82dc1d63..2ecd61b5b94820 100644 --- a/Modules/expat/winconfig.h +++ b/Modules/expat/winconfig.h @@ -6,8 +6,10 @@ \___/_/\_\ .__/ \__,_|\__| |_| XML parser - Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper <coopercc(a)users.sourceforge.net> + Copyright (c) 2002 Greg Stein <gstein(a)users.sourceforge.net> + Copyright (c) 2005 Karl Waclawek <karl(a)waclawek.net> + Copyright (c) 2017-2021 Sebastian Pipping <sebastian(a)pipping.org> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -40,17 +42,4 @@ #include <memory.h> #include <string.h> -#if defined(HAVE_EXPAT_CONFIG_H) /* e.g. MinGW */ -# include <expat_config.h> -#else /* !defined(HAVE_EXPAT_CONFIG_H) */ - -# define XML_NS 1 -# define XML_DTD 1 -# define XML_CONTEXT_BYTES 1024 - -/* we will assume all Windows platforms are little endian */ -# define BYTEORDER 1234 - -#endif /* !defined(HAVE_EXPAT_CONFIG_H) */ - #endif /* ndef WINCONFIG_H */ diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index e740f0e19c7d4a..5ba56eaea6357a 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -1,4 +1,4 @@ -/* f2d0ab6d1d4422a08cf1cf3bbdfba96b49dea42fb5ff4615e03a2a25c306e769 (2.2.8+) +/* 8539b9040d9d901366a62560a064af7cb99811335784b363abc039c5b0ebc416 (2.4.1+) __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -7,7 +7,31 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper <coopercc(a)users.sourceforge.net> + Copyright (c) 2000-2006 Fred L. Drake, Jr. <fdrake(a)users.sourceforge.net> + Copyright (c) 2001-2002 Greg Stein <gstein(a)users.sourceforge.net> + Copyright (c) 2002-2016 Karl Waclawek <karl(a)waclawek.net> + Copyright (c) 2005-2009 Steven Solie <ssolie(a)users.sourceforge.net> + Copyright (c) 2016 Eric Rahm <erahm(a)mozilla.com> + Copyright (c) 2016-2021 Sebastian Pipping <sebastian(a)pipping.org> + Copyright (c) 2016 Gaurav <g.gupta(a)samsung.com> + Copyright (c) 2016 Thomas Beutlich <tc(a)tbeu.de> + Copyright (c) 2016 Gustavo Grieco <gustavo.grieco(a)imag.fr> + Copyright (c) 2016 Pascal Cuoq <cuoq(a)trust-in-soft.com> + Copyright (c) 2016 Ed Schouten <ed(a)nuxi.nl> + Copyright (c) 2017-2018 Rhodri James <rhodri(a)wildebeest.org.uk> + Copyright (c) 2017 Václav Slavík <vaclav(a)slavik.io> + Copyright (c) 2017 Viktor Szakats <commit(a)vsz.me> + Copyright (c) 2017 Chanho Park <chanho61.park(a)samsung.com> + Copyright (c) 2017 Rolf Eike Beer <eike(a)sf-mail.de> + Copyright (c) 2017 Hans Wennborg <hans(a)chromium.org> + Copyright (c) 2018 Anton Maklakov <antmak.pub(a)gmail.com> + Copyright (c) 2018 Benjamin Peterson <benjamin(a)python.org> + Copyright (c) 2018 Marco Maggi <marco.maggi-ipsu(a)poste.it> + Copyright (c) 2018 Mariusz Zaborski <oshogbo(a)vexillium.org> + Copyright (c) 2019 David Loffredo <loffredo(a)steptools.com> + Copyright (c) 2019-2020 Ben Wagner <bungeman(a)chromium.org> + Copyright (c) 2019 Vadim Zeitlin <vadim(a)zeitlins.org> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -36,7 +60,9 @@ #ifdef _WIN32 /* force stdlib to define rand_s() */ -# define _CRT_RAND_S +# if ! defined(_CRT_RAND_S) +# define _CRT_RAND_S +# endif #endif #include <stddef.h> @@ -45,6 +71,8 @@ #include <limits.h> /* UINT_MAX */ #include <stdio.h> /* fprintf */ #include <stdlib.h> /* getenv, rand_s */ +#include <stdint.h> /* uintptr_t */ +#include <math.h> /* isnan */ #ifdef _WIN32 # define getpid GetCurrentProcessId @@ -60,9 +88,9 @@ #ifdef _WIN32 # include "winconfig.h" -#elif defined(HAVE_EXPAT_CONFIG_H) -# include <expat_config.h> -#endif /* ndef _WIN32 */ +#endif + +#include <expat_config.h> #include "ascii.h" #include "expat.h" @@ -97,14 +125,14 @@ enabled. For end user security, that is probably not what you want. \ \ Your options include: \ - * Linux + glibc >=2.25 (getrandom): HAVE_GETRANDOM, \ - * Linux + glibc <2.25 (syscall SYS_getrandom): HAVE_SYSCALL_GETRANDOM, \ + * Linux >=3.17 + glibc >=2.25 (getrandom): HAVE_GETRANDOM, \ + * Linux >=3.17 + glibc (including <2.25) (syscall SYS_getrandom): HAVE_SYSCALL_GETRANDOM, \ * BSD / macOS >=10.7 (arc4random_buf): HAVE_ARC4RANDOM_BUF, \ - * BSD / macOS <10.7 (arc4random): HAVE_ARC4RANDOM, \ + * BSD / macOS (including <10.7) (arc4random): HAVE_ARC4RANDOM, \ * libbsd (arc4random_buf): HAVE_ARC4RANDOM_BUF + HAVE_LIBBSD, \ * libbsd (arc4random): HAVE_ARC4RANDOM + HAVE_LIBBSD, \ - * Linux / BSD / macOS (/dev/urandom): XML_DEV_URANDOM \ - * Windows (rand_s): _WIN32. \ + * Linux (including <3.17) / BSD / macOS (including <10.7) (/dev/urandom): XML_DEV_URANDOM, \ + * Windows >=Vista (rand_s): _WIN32. \ \ If insist on not using any of these, bypass this error by defining \ XML_POOR_ENTROPY; you have been warned. \ @@ -119,9 +147,7 @@ # define XmlGetInternalEncoding XmlGetUtf16InternalEncoding # define XmlGetInternalEncodingNS XmlGetUtf16InternalEncodingNS # define XmlEncode XmlUtf16Encode -/* Using pointer subtraction to convert to integer type. */ -# define MUST_CONVERT(enc, s) \ - (! (enc)->isUtf16 || (((char *)(s) - (char *)NULL) & 1)) +# define MUST_CONVERT(enc, s) (! (enc)->isUtf16 || (((uintptr_t)(s)) & 1)) typedef unsigned short ICHAR; #else # define XML_ENCODE_MAX XML_UTF8_ENCODE_MAX @@ -371,6 +397,31 @@ typedef struct open_internal_entity { XML_Bool betweenDecl; /* WFC: PE Between Declarations */ } OPEN_INTERNAL_ENTITY; +enum XML_Account { + XML_ACCOUNT_DIRECT, /* bytes directly passed to the Expat parser */ + XML_ACCOUNT_ENTITY_EXPANSION, /* intermediate bytes produced during entity + expansion */ + XML_ACCOUNT_NONE /* i.e. do not account, was accounted already */ +}; + +#ifdef XML_DTD +typedef unsigned long long XmlBigCount; +typedef struct accounting { + XmlBigCount countBytesDirect; + XmlBigCount countBytesIndirect; + int debugLevel; + float maximumAmplificationFactor; // >=1.0 + unsigned long long activationThresholdBytes; +} ACCOUNTING; + +typedef struct entity_stats { + unsigned int countEverOpened; + unsigned int currentDepth; + unsigned int maximumDepthSeen; + int debugLevel; +} ENTITY_STATS; +#endif /* XML_DTD */ + typedef enum XML_Error PTRCALL Processor(XML_Parser parser, const char *start, const char *end, const char **endPtr); @@ -401,16 +452,18 @@ static enum XML_Error initializeEncoding(XML_Parser parser); static enum XML_Error doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, int tok, const char *next, const char **nextPtr, - XML_Bool haveMore, XML_Bool allowClosingDoctype); + XML_Bool haveMore, XML_Bool allowClosingDoctype, + enum XML_Account account); static enum XML_Error processInternalEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl); static enum XML_Error doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, const char *start, const char *end, const char **endPtr, - XML_Bool haveMore); + XML_Bool haveMore, enum XML_Account account); static enum XML_Error doCdataSection(XML_Parser parser, const ENCODING *, const char **startPtr, const char *end, - const char **nextPtr, XML_Bool haveMore); + const char **nextPtr, XML_Bool haveMore, + enum XML_Account account); #ifdef XML_DTD static enum XML_Error doIgnoreSection(XML_Parser parser, const ENCODING *, const char **startPtr, const char *end, @@ -420,7 +473,8 @@ static enum XML_Error doIgnoreSection(XML_Parser parser, const ENCODING *, static void freeBindings(XML_Parser parser, BINDING *bindings); static enum XML_Error storeAtts(XML_Parser parser, const ENCODING *, const char *s, TAG_NAME *tagNamePtr, - BINDING **bindingsPtr); + BINDING **bindingsPtr, + enum XML_Account account); static enum XML_Error addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, const XML_Char *uri, BINDING **bindingsPtr); @@ -429,15 +483,18 @@ static int defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *, XML_Bool isCdata, XML_Parser parser); static enum XML_Error storeAttributeValue(XML_Parser parser, const ENCODING *, XML_Bool isCdata, const char *, - const char *, STRING_POOL *); + const char *, STRING_POOL *, + enum XML_Account account); static enum XML_Error appendAttributeValue(XML_Parser parser, const ENCODING *, XML_Bool isCdata, const char *, - const char *, STRING_POOL *); + const char *, STRING_POOL *, + enum XML_Account account); static ATTRIBUTE_ID *getAttributeId(XML_Parser parser, const ENCODING *enc, const char *start, const char *end); static int setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *); static enum XML_Error storeEntityValue(XML_Parser parser, const ENCODING *enc, - const char *start, const char *end); + const char *start, const char *end, + enum XML_Account account); static int reportProcessingInstruction(XML_Parser parser, const ENCODING *enc, const char *start, const char *end); static int reportComment(XML_Parser parser, const ENCODING *enc, @@ -501,6 +558,34 @@ static XML_Parser parserCreate(const XML_Char *encodingName, static void parserInit(XML_Parser parser, const XML_Char *encodingName); +#ifdef XML_DTD +static float accountingGetCurrentAmplification(XML_Parser rootParser); +static void accountingReportStats(XML_Parser originParser, const char *epilog); +static void accountingOnAbort(XML_Parser originParser); +static void accountingReportDiff(XML_Parser rootParser, + unsigned int levelsAwayFromRootParser, + const char *before, const char *after, + ptrdiff_t bytesMore, int source_line, + enum XML_Account account); +static XML_Bool accountingDiffTolerated(XML_Parser originParser, int tok, + const char *before, const char *after, + int source_line, + enum XML_Account account); + +static void entityTrackingReportStats(XML_Parser parser, ENTITY *entity, + const char *action, int sourceLine); +static void entityTrackingOnOpen(XML_Parser parser, ENTITY *entity, + int sourceLine); +static void entityTrackingOnClose(XML_Parser parser, ENTITY *entity, + int sourceLine); + +static XML_Parser getRootParserOf(XML_Parser parser, + unsigned int *outLevelDiff); +#endif /* XML_DTD */ + +static unsigned long getDebugLevel(const char *variableName, + unsigned long defaultDebugLevel); + #define poolStart(pool) ((pool)->start) #define poolEnd(pool) ((pool)->ptr) #define poolLength(pool) ((pool)->ptr - (pool)->start) @@ -614,6 +699,10 @@ struct XML_ParserStruct { enum XML_ParamEntityParsing m_paramEntityParsing; #endif unsigned long m_hash_secret_salt; +#ifdef XML_DTD + ACCOUNTING m_accounting; + ENTITY_STATS m_entity_stats; +#endif }; #define MALLOC(parser, s) (parser->m_mem.malloc_fcn((s))) @@ -734,6 +823,15 @@ writeRandomBytes_arc4random(void *target, size_t count) { #ifdef _WIN32 +/* Provide declaration of rand_s() for MinGW-32 (not 64, which has it), + as it didn't declare it in its header prior to version 5.3.0 of its + runtime package (mingwrt, containing stdlib.h). The upstream fix + was introduced at https://osdn.net/projects/mingw/ticket/39658 . */ +# if defined(__MINGW32__) && defined(__MINGW32_VERSION) \ + && __MINGW32_VERSION < 5003000L && ! defined(__MINGW64_VERSION_MAJOR) +__declspec(dllimport) int rand_s(unsigned int *); +# endif + /* Obtain entropy on Windows using the rand_s() function which * generates cryptographically secure random numbers. Internally it * uses RtlGenRandom API which is present in Windows XP and later. @@ -789,9 +887,8 @@ gather_time_entropy(void) { static unsigned long ENTROPY_DEBUG(const char *label, unsigned long entropy) { - const char *const EXPAT_ENTROPY_DEBUG = getenv("EXPAT_ENTROPY_DEBUG"); - if (EXPAT_ENTROPY_DEBUG && ! strcmp(EXPAT_ENTROPY_DEBUG, "1")) { - fprintf(stderr, "Entropy: %s --> 0x%0*lx (%lu bytes)\n", label, + if (getDebugLevel("EXPAT_ENTROPY_DEBUG", 0) >= 1u) { + fprintf(stderr, "expat: Entropy: %s --> 0x%0*lx (%lu bytes)\n", label, (int)sizeof(entropy) * 2, entropy, (unsigned long)sizeof(entropy)); } return entropy; @@ -1053,6 +1150,18 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { parser->m_paramEntityParsing = XML_PARAM_ENTITY_PARSING_NEVER; #endif parser->m_hash_secret_salt = 0; + +#ifdef XML_DTD + memset(&parser->m_accounting, 0, sizeof(ACCOUNTING)); + parser->m_accounting.debugLevel = getDebugLevel("EXPAT_ACCOUNTING_DEBUG", 0u); + parser->m_accounting.maximumAmplificationFactor + = EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT; + parser->m_accounting.activationThresholdBytes + = EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT; + + memset(&parser->m_entity_stats, 0, sizeof(ENTITY_STATS)); + parser->m_entity_stats.debugLevel = getDebugLevel("EXPAT_ENTITY_DEBUG", 0u); +#endif } /* moves list of bindings to m_freeBindingList */ @@ -1399,6 +1508,7 @@ XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD) { parser->m_useForeignDTD = useDTD; return XML_ERROR_NONE; #else + UNUSED_P(useDTD); return XML_ERROR_FEATURE_REQUIRES_XML_DTD; #endif } @@ -1780,7 +1890,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { int nLeftOver; enum XML_Status result; /* Detect overflow (a+b > MAX <==> b > MAX-a) */ - if (len > ((XML_Size)-1) / 2 - parser->m_parseEndByteIndex) { + if ((XML_Size)len > ((XML_Size)-1) / 2 - parser->m_parseEndByteIndex) { parser->m_errorCode = XML_ERROR_NO_MEMORY; parser->m_eventPtr = parser->m_eventEndPtr = NULL; parser->m_processor = errorProcessor; @@ -1872,6 +1982,12 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { parser->m_errorCode = XML_ERROR_FINISHED; return XML_STATUS_ERROR; case XML_INITIALIZED: + /* Has someone called XML_GetBuffer successfully before? */ + if (! parser->m_bufferPtr) { + parser->m_errorCode = XML_ERROR_NO_BUFFER; + return XML_STATUS_ERROR; + } + if (parser->m_parentParser == NULL && ! startParsing(parser)) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return XML_STATUS_ERROR; @@ -2155,7 +2271,7 @@ XML_GetInputContext(XML_Parser parser, int *offset, int *size) { (void)offset; (void)size; #endif /* defined XML_CONTEXT_BYTES */ - return (char *)0; + return (const char *)0; } XML_Size XMLCALL @@ -2316,6 +2432,14 @@ XML_ErrorString(enum XML_Error code) { /* Added in 2.2.5. */ case XML_ERROR_INVALID_ARGUMENT: /* Constant added in 2.2.1, already */ return XML_L("invalid argument"); + /* Added in 2.3.0. */ + case XML_ERROR_NO_BUFFER: + return XML_L( + "a successful prior call to function XML_GetBuffer is required"); + /* Added in 2.4.0. */ + case XML_ERROR_AMPLIFICATION_LIMIT_BREACH: + return XML_L( + "limit on input amplification factor (from DTD and entities) breached"); } return NULL; } @@ -2352,41 +2476,75 @@ XML_ExpatVersionInfo(void) { const XML_Feature *XMLCALL XML_GetFeatureList(void) { - static const XML_Feature features[] - = {{XML_FEATURE_SIZEOF_XML_CHAR, XML_L("sizeof(XML_Char)"), - sizeof(XML_Char)}, - {XML_FEATURE_SIZEOF_XML_LCHAR, XML_L("sizeof(XML_LChar)"), - sizeof(XML_LChar)}, + static const XML_Feature features[] = { + {XML_FEATURE_SIZEOF_XML_CHAR, XML_L("sizeof(XML_Char)"), + sizeof(XML_Char)}, + {XML_FEATURE_SIZEOF_XML_LCHAR, XML_L("sizeof(XML_LChar)"), + sizeof(XML_LChar)}, #ifdef XML_UNICODE - {XML_FEATURE_UNICODE, XML_L("XML_UNICODE"), 0}, + {XML_FEATURE_UNICODE, XML_L("XML_UNICODE"), 0}, #endif #ifdef XML_UNICODE_WCHAR_T - {XML_FEATURE_UNICODE_WCHAR_T, XML_L("XML_UNICODE_WCHAR_T"), 0}, + {XML_FEATURE_UNICODE_WCHAR_T, XML_L("XML_UNICODE_WCHAR_T"), 0}, #endif #ifdef XML_DTD - {XML_FEATURE_DTD, XML_L("XML_DTD"), 0}, + {XML_FEATURE_DTD, XML_L("XML_DTD"), 0}, #endif #ifdef XML_CONTEXT_BYTES - {XML_FEATURE_CONTEXT_BYTES, XML_L("XML_CONTEXT_BYTES"), - XML_CONTEXT_BYTES}, + {XML_FEATURE_CONTEXT_BYTES, XML_L("XML_CONTEXT_BYTES"), + XML_CONTEXT_BYTES}, #endif #ifdef XML_MIN_SIZE - {XML_FEATURE_MIN_SIZE, XML_L("XML_MIN_SIZE"), 0}, + {XML_FEATURE_MIN_SIZE, XML_L("XML_MIN_SIZE"), 0}, #endif #ifdef XML_NS - {XML_FEATURE_NS, XML_L("XML_NS"), 0}, + {XML_FEATURE_NS, XML_L("XML_NS"), 0}, #endif #ifdef XML_LARGE_SIZE - {XML_FEATURE_LARGE_SIZE, XML_L("XML_LARGE_SIZE"), 0}, + {XML_FEATURE_LARGE_SIZE, XML_L("XML_LARGE_SIZE"), 0}, #endif #ifdef XML_ATTR_INFO - {XML_FEATURE_ATTR_INFO, XML_L("XML_ATTR_INFO"), 0}, + {XML_FEATURE_ATTR_INFO, XML_L("XML_ATTR_INFO"), 0}, +#endif +#ifdef XML_DTD + /* Added in Expat 2.4.0. */ + {XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT, + XML_L("XML_BLAP_MAX_AMP"), + (long int) + EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT}, + {XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT, + XML_L("XML_BLAP_ACT_THRES"), + EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT}, #endif - {XML_FEATURE_END, NULL, 0}}; + {XML_FEATURE_END, NULL, 0}}; return features; } +#ifdef XML_DTD +XML_Bool XMLCALL +XML_SetBillionLaughsAttackProtectionMaximumAmplification( + XML_Parser parser, float maximumAmplificationFactor) { + if ((parser == NULL) || (parser->m_parentParser != NULL) + || isnan(maximumAmplificationFactor) + || (maximumAmplificationFactor < 1.0f)) { + return XML_FALSE; + } + parser->m_accounting.maximumAmplificationFactor = maximumAmplificationFactor; + return XML_TRUE; +} + +XML_Bool XMLCALL +XML_SetBillionLaughsAttackProtectionActivationThreshold( + XML_Parser parser, unsigned long long activationThresholdBytes) { + if ((parser == NULL) || (parser->m_parentParser != NULL)) { + return XML_FALSE; + } + parser->m_accounting.activationThresholdBytes = activationThresholdBytes; + return XML_TRUE; +} +#endif /* XML_DTD */ + /* Initially tag->rawName always points into the parse buffer; for those TAG instances opened while the current parse buffer was processed, and not yet closed, we need to store tag->rawName in a more @@ -2439,9 +2597,9 @@ storeRawNames(XML_Parser parser) { static enum XML_Error PTRCALL contentProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { - enum XML_Error result - = doContent(parser, 0, parser->m_encoding, start, end, endPtr, - (XML_Bool)! parser->m_parsingStatus.finalBuffer); + enum XML_Error result = doContent( + parser, 0, parser->m_encoding, start, end, endPtr, + (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_ACCOUNT_DIRECT); if (result == XML_ERROR_NONE) { if (! storeRawNames(parser)) return XML_ERROR_NO_MEMORY; @@ -2466,6 +2624,14 @@ externalEntityInitProcessor2(XML_Parser parser, const char *start, int tok = XmlContentTok(parser->m_encoding, start, end, &next); switch (tok) { case XML_TOK_BOM: +#ifdef XML_DTD + if (! accountingDiffTolerated(parser, tok, start, next, __LINE__, + XML_ACCOUNT_DIRECT)) { + accountingOnAbort(parser); + return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + } +#endif /* XML_DTD */ + /* If we are at the end of the buffer, this would cause the next stage, i.e. externalEntityInitProcessor3, to pass control directly to doContent (by detecting XML_TOK_NONE) without processing any xml text @@ -2503,6 +2669,10 @@ externalEntityInitProcessor3(XML_Parser parser, const char *start, const char *next = start; /* XmlContentTok doesn't always set the last arg */ parser->m_eventPtr = start; tok = XmlContentTok(parser->m_encoding, start, end, &next); + /* Note: These bytes are accounted later in: + - processXmlDecl + - externalEntityContentProcessor + */ parser->m_eventEndPtr = next; switch (tok) { @@ -2544,7 +2714,8 @@ externalEntityContentProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { enum XML_Error result = doContent(parser, 1, parser->m_encoding, start, end, endPtr, - (XML_Bool)! parser->m_parsingStatus.finalBuffer); + (XML_Bool)! parser->m_parsingStatus.finalBuffer, + XML_ACCOUNT_ENTITY_EXPANSION); if (result == XML_ERROR_NONE) { if (! storeRawNames(parser)) return XML_ERROR_NO_MEMORY; @@ -2555,7 +2726,7 @@ externalEntityContentProcessor(XML_Parser parser, const char *start, static enum XML_Error doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, const char *s, const char *end, const char **nextPtr, - XML_Bool haveMore) { + XML_Bool haveMore, enum XML_Account account) { /* save one level of indirection */ DTD *const dtd = parser->m_dtd; @@ -2573,6 +2744,17 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, for (;;) { const char *next = s; /* XmlContentTok doesn't always set the last arg */ int tok = XmlContentTok(enc, s, end, &next); +#ifdef XML_DTD + const char *accountAfter + = ((tok == XML_TOK_TRAILING_RSQB) || (tok == XML_TOK_TRAILING_CR)) + ? (haveMore ? s /* i.e. 0 bytes */ : end) + : next; + if (! accountingDiffTolerated(parser, tok, s, accountAfter, __LINE__, + account)) { + accountingOnAbort(parser); + return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + } +#endif *eventEndPP = next; switch (tok) { case XML_TOK_TRAILING_CR: @@ -2628,6 +2810,14 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, XML_Char ch = (XML_Char)XmlPredefinedEntityName( enc, s + enc->minBytesPerChar, next - enc->minBytesPerChar); if (ch) { +#ifdef XML_DTD + /* NOTE: We are replacing 4-6 characters original input for 1 character + * so there is no amplification and hence recording without + * protection. */ + accountingDiffTolerated(parser, tok, (char *)&ch, + ((char *)&ch) + sizeof(XML_Char), __LINE__, + XML_ACCOUNT_ENTITY_EXPANSION); +#endif /* XML_DTD */ if (parser->m_characterDataHandler) parser->m_characterDataHandler(parser->m_handlerArg, &ch, 1); else if (parser->m_defaultHandler) @@ -2746,7 +2936,8 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, } tag->name.str = (XML_Char *)tag->buf; *toPtr = XML_T('\0'); - result = storeAtts(parser, enc, s, &(tag->name), &(tag->bindings)); + result + = storeAtts(parser, enc, s, &(tag->name), &(tag->bindings), account); if (result) return result; if (parser->m_startElementHandler) @@ -2770,7 +2961,8 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, if (! name.str) return XML_ERROR_NO_MEMORY; poolFinish(&parser->m_tempPool); - result = storeAtts(parser, enc, s, &name, &bindings); + result = storeAtts(parser, enc, s, &name, &bindings, + XML_ACCOUNT_NONE /* token spans whole start tag */); if (result != XML_ERROR_NONE) { freeBindings(parser, bindings); return result; @@ -2905,7 +3097,8 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, /* END disabled code */ else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); - result = doCdataSection(parser, enc, &next, end, nextPtr, haveMore); + result + = doCdataSection(parser, enc, &next, end, nextPtr, haveMore, account); if (result != XML_ERROR_NONE) return result; else if (! next) { @@ -3034,7 +3227,8 @@ freeBindings(XML_Parser parser, BINDING *bindings) { */ static enum XML_Error storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, - TAG_NAME *tagNamePtr, BINDING **bindingsPtr) { + TAG_NAME *tagNamePtr, BINDING **bindingsPtr, + enum XML_Account account) { DTD *const dtd = parser->m_dtd; /* save one level of indirection */ ELEMENT_TYPE *elementType; int nDefaultAtts; @@ -3144,7 +3338,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, /* normalize the attribute value */ result = storeAttributeValue( parser, enc, isCdata, parser->m_atts[i].valuePtr, - parser->m_atts[i].valueEnd, &parser->m_tempPool); + parser->m_atts[i].valueEnd, &parser->m_tempPool, account); if (result) return result; appAtts[attIndex] = poolStart(&parser->m_tempPool); @@ -3533,9 +3727,9 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, static enum XML_Error PTRCALL cdataSectionProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { - enum XML_Error result - = doCdataSection(parser, parser->m_encoding, &start, end, endPtr, - (XML_Bool)! parser->m_parsingStatus.finalBuffer); + enum XML_Error result = doCdataSection( + parser, parser->m_encoding, &start, end, endPtr, + (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_ACCOUNT_DIRECT); if (result != XML_ERROR_NONE) return result; if (start) { @@ -3555,7 +3749,8 @@ cdataSectionProcessor(XML_Parser parser, const char *start, const char *end, */ static enum XML_Error doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, - const char *end, const char **nextPtr, XML_Bool haveMore) { + const char *end, const char **nextPtr, XML_Bool haveMore, + enum XML_Account account) { const char *s = *startPtr; const char **eventPP; const char **eventEndPP; @@ -3571,8 +3766,16 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, *startPtr = NULL; for (;;) { - const char *next; + const char *next = s; /* in case of XML_TOK_NONE or XML_TOK_PARTIAL */ int tok = XmlCdataSectionTok(enc, s, end, &next); +#ifdef XML_DTD + if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, account)) { + accountingOnAbort(parser); + return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + } +#else + UNUSED_P(account); +#endif *eventEndPP = next; switch (tok) { case XML_TOK_CDATA_SECT_CLOSE: @@ -3689,7 +3892,7 @@ ignoreSectionProcessor(XML_Parser parser, const char *start, const char *end, static enum XML_Error doIgnoreSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, const char *end, const char **nextPtr, XML_Bool haveMore) { - const char *next; + const char *next = *startPtr; /* in case of XML_TOK_NONE or XML_TOK_PARTIAL */ int tok; const char *s = *startPtr; const char **eventPP; @@ -3717,6 +3920,13 @@ doIgnoreSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, *eventPP = s; *startPtr = NULL; tok = XmlIgnoreSectionTok(enc, s, end, &next); +# ifdef XML_DTD + if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, + XML_ACCOUNT_DIRECT)) { + accountingOnAbort(parser); + return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + } +# endif *eventEndPP = next; switch (tok) { case XML_TOK_IGNORE_SECT: @@ -3801,6 +4011,15 @@ processXmlDecl(XML_Parser parser, int isGeneralTextEntity, const char *s, const char *versionend; const XML_Char *storedversion = NULL; int standalone = -1; + +#ifdef XML_DTD + if (! accountingDiffTolerated(parser, XML_TOK_XML_DECL, s, next, __LINE__, + XML_ACCOUNT_DIRECT)) { + accountingOnAbort(parser); + return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + } +#endif + if (! (parser->m_ns ? XmlParseXmlDeclNS : XmlParseXmlDecl)( isGeneralTextEntity, parser->m_encoding, s, next, &parser->m_eventPtr, &version, &versionend, &encodingName, &newEncoding, &standalone)) { @@ -3950,6 +4169,10 @@ entityValueInitProcessor(XML_Parser parser, const char *s, const char *end, for (;;) { tok = XmlPrologTok(parser->m_encoding, start, end, &next); + /* Note: Except for XML_TOK_BOM below, these bytes are accounted later in: + - storeEntityValue + - processXmlDecl + */ parser->m_eventEndPtr = next; if (tok <= 0) { if (! parser->m_parsingStatus.finalBuffer && tok != XML_TOK_INVALID) { @@ -3968,7 +4191,8 @@ entityValueInitProcessor(XML_Parser parser, const char *s, const char *end, break; } /* found end of entity value - can store it now */ - return storeEntityValue(parser, parser->m_encoding, s, end); + return storeEntityValue(parser, parser->m_encoding, s, end, + XML_ACCOUNT_DIRECT); } else if (tok == XML_TOK_XML_DECL) { enum XML_Error result; result = processXmlDecl(parser, 0, start, next); @@ -3995,6 +4219,14 @@ entityValueInitProcessor(XML_Parser parser, const char *s, const char *end, */ else if (tok == XML_TOK_BOM && next == end && ! parser->m_parsingStatus.finalBuffer) { +# ifdef XML_DTD + if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, + XML_ACCOUNT_DIRECT)) { + accountingOnAbort(parser); + return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + } +# endif + *nextPtr = next; return XML_ERROR_NONE; } @@ -4037,16 +4269,24 @@ externalParEntProcessor(XML_Parser parser, const char *s, const char *end, } /* This would cause the next stage, i.e. doProlog to be passed XML_TOK_BOM. However, when parsing an external subset, doProlog will not accept a BOM - as valid, and report a syntax error, so we have to skip the BOM + as valid, and report a syntax error, so we have to skip the BOM, and + account for the BOM bytes. */ else if (tok == XML_TOK_BOM) { + if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, + XML_ACCOUNT_DIRECT)) { + accountingOnAbort(parser); + return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + } + s = next; tok = XmlPrologTok(parser->m_encoding, s, end, &next); } parser->m_processor = prologProcessor; return doProlog(parser, parser->m_encoding, s, end, tok, next, nextPtr, - (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE); + (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE, + XML_ACCOUNT_DIRECT); } static enum XML_Error PTRCALL @@ -4059,6 +4299,9 @@ entityValueProcessor(XML_Parser parser, const char *s, const char *end, for (;;) { tok = XmlPrologTok(enc, start, end, &next); + /* Note: These bytes are accounted later in: + - storeEntityValue + */ if (tok <= 0) { if (! parser->m_parsingStatus.finalBuffer && tok != XML_TOK_INVALID) { *nextPtr = s; @@ -4076,7 +4319,7 @@ entityValueProcessor(XML_Parser parser, const char *s, const char *end, break; } /* found end of entity value - can store it now */ - return storeEntityValue(parser, enc, s, end); + return storeEntityValue(parser, enc, s, end, XML_ACCOUNT_DIRECT); } start = next; } @@ -4090,13 +4333,14 @@ prologProcessor(XML_Parser parser, const char *s, const char *end, const char *next = s; int tok = XmlPrologTok(parser->m_encoding, s, end, &next); return doProlog(parser, parser->m_encoding, s, end, tok, next, nextPtr, - (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE); + (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE, + XML_ACCOUNT_DIRECT); } static enum XML_Error doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, int tok, const char *next, const char **nextPtr, XML_Bool haveMore, - XML_Bool allowClosingDoctype) { + XML_Bool allowClosingDoctype, enum XML_Account account) { #ifdef XML_DTD static const XML_Char externalSubsetName[] = {ASCII_HASH, '\0'}; #endif /* XML_DTD */ @@ -4123,6 +4367,10 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, static const XML_Char enumValueSep[] = {ASCII_PIPE, '\0'}; static const XML_Char enumValueStart[] = {ASCII_LPAREN, '\0'}; +#ifndef XML_DTD + UNUSED_P(account); +#endif + /* save one level of indirection */ DTD *const dtd = parser->m_dtd; @@ -4187,6 +4435,19 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } } role = XmlTokenRole(&parser->m_prologState, tok, s, next, enc); +#ifdef XML_DTD + switch (role) { + case XML_ROLE_INSTANCE_START: // bytes accounted in contentProcessor + case XML_ROLE_XML_DECL: // bytes accounted in processXmlDecl + case XML_ROLE_TEXT_DECL: // bytes accounted in processXmlDecl + break; + default: + if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, account)) { + accountingOnAbort(parser); + return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + } + } +#endif switch (role) { case XML_ROLE_XML_DECL: { enum XML_Error result = processXmlDecl(parser, 0, s, next); @@ -4462,7 +4723,8 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, const XML_Char *attVal; enum XML_Error result = storeAttributeValue( parser, enc, parser->m_declAttributeIsCdata, - s + enc->minBytesPerChar, next - enc->minBytesPerChar, &dtd->pool); + s + enc->minBytesPerChar, next - enc->minBytesPerChar, &dtd->pool, + XML_ACCOUNT_NONE); if (result) return result; attVal = poolStart(&dtd->pool); @@ -4495,8 +4757,9 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, break; case XML_ROLE_ENTITY_VALUE: if (dtd->keepProcessing) { - enum XML_Error result = storeEntityValue( - parser, enc, s + enc->minBytesPerChar, next - enc->minBytesPerChar); + enum XML_Error result + = storeEntityValue(parser, enc, s + enc->minBytesPerChar, + next - enc->minBytesPerChar, XML_ACCOUNT_NONE); if (parser->m_declEntity) { parser->m_declEntity->textPtr = poolStart(&dtd->entityValuePool); parser->m_declEntity->textLen @@ -4886,12 +5149,15 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (parser->m_externalEntityRefHandler) { dtd->paramEntityRead = XML_FALSE; entity->open = XML_TRUE; + entityTrackingOnOpen(parser, entity, __LINE__); if (! parser->m_externalEntityRefHandler( parser->m_externalEntityRefHandlerArg, 0, entity->base, entity->systemId, entity->publicId)) { + entityTrackingOnClose(parser, entity, __LINE__); entity->open = XML_FALSE; return XML_ERROR_EXTERNAL_ENTITY_HANDLING; } + entityTrackingOnClose(parser, entity, __LINE__); entity->open = XML_FALSE; handleDefault = XML_FALSE; if (! dtd->paramEntityRead) { @@ -5089,6 +5355,13 @@ epilogProcessor(XML_Parser parser, const char *s, const char *end, for (;;) { const char *next = NULL; int tok = XmlPrologTok(parser->m_encoding, s, end, &next); +#ifdef XML_DTD + if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, + XML_ACCOUNT_DIRECT)) { + accountingOnAbort(parser); + return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + } +#endif parser->m_eventEndPtr = next; switch (tok) { /* report partial linebreak - it might be the last token */ @@ -5162,6 +5435,9 @@ processInternalEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl) { return XML_ERROR_NO_MEMORY; } entity->open = XML_TRUE; +#ifdef XML_DTD + entityTrackingOnOpen(parser, entity, __LINE__); +#endif entity->processed = 0; openEntity->next = parser->m_openInternalEntities; parser->m_openInternalEntities = openEntity; @@ -5170,8 +5446,8 @@ processInternalEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl) { openEntity->betweenDecl = betweenDecl; openEntity->internalEventPtr = NULL; openEntity->internalEventEndPtr = NULL; - textStart = (char *)entity->textPtr; - textEnd = (char *)(entity->textPtr + entity->textLen); + textStart = (const char *)entity->textPtr; + textEnd = (const char *)(entity->textPtr + entity->textLen); /* Set a safe default value in case 'next' does not get set */ next = textStart; @@ -5180,17 +5456,22 @@ processInternalEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl) { int tok = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next); result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd, - tok, next, &next, XML_FALSE, XML_FALSE); + tok, next, &next, XML_FALSE, XML_FALSE, + XML_ACCOUNT_ENTITY_EXPANSION); } else #endif /* XML_DTD */ result = doContent(parser, parser->m_tagLevel, parser->m_internalEncoding, - textStart, textEnd, &next, XML_FALSE); + textStart, textEnd, &next, XML_FALSE, + XML_ACCOUNT_ENTITY_EXPANSION); if (result == XML_ERROR_NONE) { if (textEnd != next && parser->m_parsingStatus.parsing == XML_SUSPENDED) { entity->processed = (int)(next - textStart); parser->m_processor = internalEntityProcessor; } else { +#ifdef XML_DTD + entityTrackingOnClose(parser, entity, __LINE__); +#endif /* XML_DTD */ entity->open = XML_FALSE; parser->m_openInternalEntities = openEntity->next; /* put openEntity back in list of free instances */ @@ -5213,8 +5494,8 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, return XML_ERROR_UNEXPECTED_STATE; entity = openEntity->entity; - textStart = ((char *)entity->textPtr) + entity->processed; - textEnd = (char *)(entity->textPtr + entity->textLen); + textStart = ((const char *)entity->textPtr) + entity->processed; + textEnd = (const char *)(entity->textPtr + entity->textLen); /* Set a safe default value in case 'next' does not get set */ next = textStart; @@ -5223,20 +5504,24 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, int tok = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next); result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd, - tok, next, &next, XML_FALSE, XML_TRUE); + tok, next, &next, XML_FALSE, XML_TRUE, + XML_ACCOUNT_ENTITY_EXPANSION); } else #endif /* XML_DTD */ result = doContent(parser, openEntity->startTagLevel, parser->m_internalEncoding, textStart, textEnd, &next, - XML_FALSE); + XML_FALSE, XML_ACCOUNT_ENTITY_EXPANSION); if (result != XML_ERROR_NONE) return result; else if (textEnd != next && parser->m_parsingStatus.parsing == XML_SUSPENDED) { - entity->processed = (int)(next - (char *)entity->textPtr); + entity->processed = (int)(next - (const char *)entity->textPtr); return result; } else { +#ifdef XML_DTD + entityTrackingOnClose(parser, entity, __LINE__); +#endif entity->open = XML_FALSE; parser->m_openInternalEntities = openEntity->next; /* put openEntity back in list of free instances */ @@ -5250,7 +5535,8 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, parser->m_processor = prologProcessor; tok = XmlPrologTok(parser->m_encoding, s, end, &next); return doProlog(parser, parser->m_encoding, s, end, tok, next, nextPtr, - (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE); + (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE, + XML_ACCOUNT_DIRECT); } else #endif /* XML_DTD */ { @@ -5258,7 +5544,8 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, /* see externalEntityContentProcessor vs contentProcessor */ return doContent(parser, parser->m_parentParser ? 1 : 0, parser->m_encoding, s, end, nextPtr, - (XML_Bool)! parser->m_parsingStatus.finalBuffer); + (XML_Bool)! parser->m_parsingStatus.finalBuffer, + XML_ACCOUNT_DIRECT); } } @@ -5273,9 +5560,10 @@ errorProcessor(XML_Parser parser, const char *s, const char *end, static enum XML_Error storeAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, - const char *ptr, const char *end, STRING_POOL *pool) { + const char *ptr, const char *end, STRING_POOL *pool, + enum XML_Account account) { enum XML_Error result - = appendAttributeValue(parser, enc, isCdata, ptr, end, pool); + = appendAttributeValue(parser, enc, isCdata, ptr, end, pool, account); if (result) return result; if (! isCdata && poolLength(pool) && poolLastChar(pool) == 0x20) @@ -5287,11 +5575,23 @@ storeAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, static enum XML_Error appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, - const char *ptr, const char *end, STRING_POOL *pool) { + const char *ptr, const char *end, STRING_POOL *pool, + enum XML_Account account) { DTD *const dtd = parser->m_dtd; /* save one level of indirection */ +#ifndef XML_DTD + UNUSED_P(account); +#endif + for (;;) { - const char *next; + const char *next + = ptr; /* XmlAttributeValueTok doesn't always set the last arg */ int tok = XmlAttributeValueTok(enc, ptr, end, &next); +#ifdef XML_DTD + if (! accountingDiffTolerated(parser, tok, ptr, next, __LINE__, account)) { + accountingOnAbort(parser); + return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + } +#endif switch (tok) { case XML_TOK_NONE: return XML_ERROR_NONE; @@ -5351,6 +5651,14 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, XML_Char ch = (XML_Char)XmlPredefinedEntityName( enc, ptr + enc->minBytesPerChar, next - enc->minBytesPerChar); if (ch) { +#ifdef XML_DTD + /* NOTE: We are replacing 4-6 characters original input for 1 character + * so there is no amplification and hence recording without + * protection. */ + accountingDiffTolerated(parser, tok, (char *)&ch, + ((char *)&ch) + sizeof(XML_Char), __LINE__, + XML_ACCOUNT_ENTITY_EXPANSION); +#endif /* XML_DTD */ if (! poolAppendChar(pool, ch)) return XML_ERROR_NO_MEMORY; break; @@ -5428,9 +5736,16 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, enum XML_Error result; const XML_Char *textEnd = entity->textPtr + entity->textLen; entity->open = XML_TRUE; +#ifdef XML_DTD + entityTrackingOnOpen(parser, entity, __LINE__); +#endif result = appendAttributeValue(parser, parser->m_internalEncoding, - isCdata, (char *)entity->textPtr, - (char *)textEnd, pool); + isCdata, (const char *)entity->textPtr, + (const char *)textEnd, pool, + XML_ACCOUNT_ENTITY_EXPANSION); +#ifdef XML_DTD + entityTrackingOnClose(parser, entity, __LINE__); +#endif entity->open = XML_FALSE; if (result) return result; @@ -5460,13 +5775,16 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, static enum XML_Error storeEntityValue(XML_Parser parser, const ENCODING *enc, - const char *entityTextPtr, const char *entityTextEnd) { + const char *entityTextPtr, const char *entityTextEnd, + enum XML_Account account) { DTD *const dtd = parser->m_dtd; /* save one level of indirection */ STRING_POOL *pool = &(dtd->entityValuePool); enum XML_Error result = XML_ERROR_NONE; #ifdef XML_DTD int oldInEntityValue = parser->m_prologState.inEntityValue; parser->m_prologState.inEntityValue = 1; +#else + UNUSED_P(account); #endif /* XML_DTD */ /* never return Null for the value argument in EntityDeclHandler, since this would indicate an external entity; therefore we @@ -5477,8 +5795,19 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, } for (;;) { - const char *next; + const char *next + = entityTextPtr; /* XmlEntityValueTok doesn't always set the last arg */ int tok = XmlEntityValueTok(enc, entityTextPtr, entityTextEnd, &next); + +#ifdef XML_DTD + if (! accountingDiffTolerated(parser, tok, entityTextPtr, next, __LINE__, + account)) { + accountingOnAbort(parser); + result = XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + goto endEntityValue; + } +#endif + switch (tok) { case XML_TOK_PARAM_ENTITY_REF: #ifdef XML_DTD @@ -5514,13 +5843,16 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, if (parser->m_externalEntityRefHandler) { dtd->paramEntityRead = XML_FALSE; entity->open = XML_TRUE; + entityTrackingOnOpen(parser, entity, __LINE__); if (! parser->m_externalEntityRefHandler( parser->m_externalEntityRefHandlerArg, 0, entity->base, entity->systemId, entity->publicId)) { + entityTrackingOnClose(parser, entity, __LINE__); entity->open = XML_FALSE; result = XML_ERROR_EXTERNAL_ENTITY_HANDLING; goto endEntityValue; } + entityTrackingOnClose(parser, entity, __LINE__); entity->open = XML_FALSE; if (! dtd->paramEntityRead) dtd->keepProcessing = dtd->standalone; @@ -5528,9 +5860,12 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, dtd->keepProcessing = dtd->standalone; } else { entity->open = XML_TRUE; + entityTrackingOnOpen(parser, entity, __LINE__); result = storeEntityValue( - parser, parser->m_internalEncoding, (char *)entity->textPtr, - (char *)(entity->textPtr + entity->textLen)); + parser, parser->m_internalEncoding, (const char *)entity->textPtr, + (const char *)(entity->textPtr + entity->textLen), + XML_ACCOUNT_ENTITY_EXPANSION); + entityTrackingOnClose(parser, entity, __LINE__); entity->open = XML_FALSE; if (result) goto endEntityValue; @@ -6485,7 +6820,7 @@ hashTableInit(HASH_TABLE *p, const XML_Memory_Handling_Suite *ms) { static void FASTCALL hashTableIterInit(HASH_TABLE_ITER *iter, const HASH_TABLE *table) { iter->p = table->v; - iter->end = iter->p + table->size; + iter->end = iter->p ? iter->p + table->size : NULL; } static NAMED *FASTCALL @@ -6891,3 +7226,755 @@ copyString(const XML_Char *s, const XML_Memory_Handling_Suite *memsuite) { memcpy(result, s, charsRequired * sizeof(XML_Char)); return result; } + +#ifdef XML_DTD + +static float +accountingGetCurrentAmplification(XML_Parser rootParser) { + const XmlBigCount countBytesOutput + = rootParser->m_accounting.countBytesDirect + + rootParser->m_accounting.countBytesIndirect; + const float amplificationFactor + = rootParser->m_accounting.countBytesDirect + ? (countBytesOutput + / (float)(rootParser->m_accounting.countBytesDirect)) + : 1.0f; + assert(! rootParser->m_parentParser); + return amplificationFactor; +} + +static void +accountingReportStats(XML_Parser originParser, const char *epilog) { + const XML_Parser rootParser = getRootParserOf(originParser, NULL); + assert(! rootParser->m_parentParser); + + if (rootParser->m_accounting.debugLevel < 1) { + return; + } + + const float amplificationFactor + = accountingGetCurrentAmplification(rootParser); + fprintf(stderr, + "expat: Accounting(%p): Direct " EXPAT_FMT_ULL( + "10") ", indirect " EXPAT_FMT_ULL("10") ", amplification %8.2f%s", + (void *)rootParser, rootParser->m_accounting.countBytesDirect, + rootParser->m_accounting.countBytesIndirect, + (double)amplificationFactor, epilog); +} + +static void +accountingOnAbort(XML_Parser originParser) { + accountingReportStats(originParser, " ABORTING\n"); +} + +static void +accountingReportDiff(XML_Parser rootParser, + unsigned int levelsAwayFromRootParser, const char *before, + const char *after, ptrdiff_t bytesMore, int source_line, + enum XML_Account account) { + assert(! rootParser->m_parentParser); + + fprintf(stderr, + " (+" EXPAT_FMT_PTRDIFF_T("6") " bytes %s|%d, xmlparse.c:%d) %*s\"", + bytesMore, (account == XML_ACCOUNT_DIRECT) ? "DIR" : "EXP", + levelsAwayFromRootParser, source_line, 10, ""); + + const char ellipis[] = "[..]"; + const size_t ellipsisLength = sizeof(ellipis) /* because compile-time */ - 1; + const unsigned int contextLength = 10; + + /* Note: Performance is of no concern here */ + const char *walker = before; + if ((rootParser->m_accounting.debugLevel >= 3) + || (after - before) + <= (ptrdiff_t)(contextLength + ellipsisLength + contextLength)) { + for (; walker < after; walker++) { + fprintf(stderr, "%s", unsignedCharToPrintable(walker[0])); + } + } else { + for (; walker < before + contextLength; walker++) { + fprintf(stderr, "%s", unsignedCharToPrintable(walker[0])); + } + fprintf(stderr, ellipis); + walker = after - contextLength; + for (; walker < after; walker++) { + fprintf(stderr, "%s", unsignedCharToPrintable(walker[0])); + } + } + fprintf(stderr, "\"\n"); +} + +static XML_Bool +accountingDiffTolerated(XML_Parser originParser, int tok, const char *before, + const char *after, int source_line, + enum XML_Account account) { + /* Note: We need to check the token type *first* to be sure that + * we can even access variable <after>, safely. + * E.g. for XML_TOK_NONE <after> may hold an invalid pointer. */ + switch (tok) { + case XML_TOK_INVALID: + case XML_TOK_PARTIAL: + case XML_TOK_PARTIAL_CHAR: + case XML_TOK_NONE: + return XML_TRUE; + } + + if (account == XML_ACCOUNT_NONE) + return XML_TRUE; /* because these bytes have been accounted for, already */ + + unsigned int levelsAwayFromRootParser; + const XML_Parser rootParser + = getRootParserOf(originParser, &levelsAwayFromRootParser); + assert(! rootParser->m_parentParser); + + const int isDirect + = (account == XML_ACCOUNT_DIRECT) && (originParser == rootParser); + const ptrdiff_t bytesMore = after - before; + + XmlBigCount *const additionTarget + = isDirect ? &rootParser->m_accounting.countBytesDirect + : &rootParser->m_accounting.countBytesIndirect; + + /* Detect and avoid integer overflow */ + if (*additionTarget > (XmlBigCount)(-1) - (XmlBigCount)bytesMore) + return XML_FALSE; + *additionTarget += bytesMore; + + const XmlBigCount countBytesOutput + = rootParser->m_accounting.countBytesDirect + + rootParser->m_accounting.countBytesIndirect; + const float amplificationFactor + = accountingGetCurrentAmplification(rootParser); + const XML_Bool tolerated + = (countBytesOutput < rootParser->m_accounting.activationThresholdBytes) + || (amplificationFactor + <= rootParser->m_accounting.maximumAmplificationFactor); + + if (rootParser->m_accounting.debugLevel >= 2) { + accountingReportStats(rootParser, ""); + accountingReportDiff(rootParser, levelsAwayFromRootParser, before, after, + bytesMore, source_line, account); + } + + return tolerated; +} + +unsigned long long +testingAccountingGetCountBytesDirect(XML_Parser parser) { + if (! parser) + return 0; + return parser->m_accounting.countBytesDirect; +} + +unsigned long long +testingAccountingGetCountBytesIndirect(XML_Parser parser) { + if (! parser) + return 0; + return parser->m_accounting.countBytesIndirect; +} + +static void +entityTrackingReportStats(XML_Parser rootParser, ENTITY *entity, + const char *action, int sourceLine) { + assert(! rootParser->m_parentParser); + if (rootParser->m_entity_stats.debugLevel < 1) + return; + +# if defined(XML_UNICODE) + const char *const entityName = "[..]"; +# else + const char *const entityName = entity->name; +# endif + + fprintf( + stderr, + "expat: Entities(%p): Count %9d, depth %2d/%2d %*s%s%s; %s length %d (xmlparse.c:%d)\n", + (void *)rootParser, rootParser->m_entity_stats.countEverOpened, + rootParser->m_entity_stats.currentDepth, + rootParser->m_entity_stats.maximumDepthSeen, + (rootParser->m_entity_stats.currentDepth - 1) * 2, "", + entity->is_param ? "%" : "&", entityName, action, entity->textLen, + sourceLine); +} + +static void +entityTrackingOnOpen(XML_Parser originParser, ENTITY *entity, int sourceLine) { + const XML_Parser rootParser = getRootParserOf(originParser, NULL); + assert(! rootParser->m_parentParser); + + rootParser->m_entity_stats.countEverOpened++; + rootParser->m_entity_stats.currentDepth++; + if (rootParser->m_entity_stats.currentDepth + > rootParser->m_entity_stats.maximumDepthSeen) { + rootParser->m_entity_stats.maximumDepthSeen++; + } + + entityTrackingReportStats(rootParser, entity, "OPEN ", sourceLine); +} + +static void +entityTrackingOnClose(XML_Parser originParser, ENTITY *entity, int sourceLine) { + const XML_Parser rootParser = getRootParserOf(originParser, NULL); + assert(! rootParser->m_parentParser); + + entityTrackingReportStats(rootParser, entity, "CLOSE", sourceLine); + rootParser->m_entity_stats.currentDepth--; +} + +static XML_Parser +getRootParserOf(XML_Parser parser, unsigned int *outLevelDiff) { + XML_Parser rootParser = parser; + unsigned int stepsTakenUpwards = 0; + while (rootParser->m_parentParser) { + rootParser = rootParser->m_parentParser; + stepsTakenUpwards++; + } + assert(! rootParser->m_parentParser); + if (outLevelDiff != NULL) { + *outLevelDiff = stepsTakenUpwards; + } + return rootParser; +} + +const char * +unsignedCharToPrintable(unsigned char c) { + switch (c) { + case 0: + return "\\0"; + case 1: + return "\\x1"; + case 2: + return "\\x2"; + case 3: + return "\\x3"; + case 4: + return "\\x4"; + case 5: + return "\\x5"; + case 6: + return "\\x6"; + case 7: + return "\\x7"; + case 8: + return "\\x8"; + case 9: + return "\\t"; + case 10: + return "\\n"; + case 11: + return "\\xB"; + case 12: + return "\\xC"; + case 13: + return "\\r"; + case 14: + return "\\xE"; + case 15: + return "\\xF"; + case 16: + return "\\x10"; + case 17: + return "\\x11"; + case 18: + return "\\x12"; + case 19: + return "\\x13"; + case 20: + return "\\x14"; + case 21: + return "\\x15"; + case 22: + return "\\x16"; + case 23: + return "\\x17"; + case 24: + return "\\x18"; + case 25: + return "\\x19"; + case 26: + return "\\x1A"; + case 27: + return "\\x1B"; + case 28: + return "\\x1C"; + case 29: + return "\\x1D"; + case 30: + return "\\x1E"; + case 31: + return "\\x1F"; + case 32: + return " "; + case 33: + return "!"; + case 34: + return "\\\""; + case 35: + return "#"; + case 36: + return "$"; + case 37: + return "%"; + case 38: + return "&"; + case 39: + return "'"; + case 40: + return "("; + case 41: + return ")"; + case 42: + return "*"; + case 43: + return "+"; + case 44: + return ","; + case 45: + return "-"; + case 46: + return "."; + case 47: + return "/"; + case 48: + return "0"; + case 49: + return "1"; + case 50: + return "2"; + case 51: + return "3"; + case 52: + return "4"; + case 53: + return "5"; + case 54: + return "6"; + case 55: + return "7"; + case 56: + return "8"; + case 57: + return "9"; + case 58: + return ":"; + case 59: + return ";"; + case 60: + return "<"; + case 61: + return "="; + case 62: + return ">"; + case 63: + return "?"; + case 64: + return "@"; + case 65: + return "A"; + case 66: + return "B"; + case 67: + return "C"; + case 68: + return "D"; + case 69: + return "E"; + case 70: + return "F"; + case 71: + return "G"; + case 72: + return "H"; + case 73: + return "I"; + case 74: + return "J"; + case 75: + return "K"; + case 76: + return "L"; + case 77: + return "M"; + case 78: + return "N"; + case 79: + return "O"; + case 80: + return "P"; + case 81: + return "Q"; + case 82: + return "R"; + case 83: + return "S"; + case 84: + return "T"; + case 85: + return "U"; + case 86: + return "V"; + case 87: + return "W"; + case 88: + return "X"; + case 89: + return "Y"; + case 90: + return "Z"; + case 91: + return "["; + case 92: + return "\\\\"; + case 93: + return "]"; + case 94: + return "^"; + case 95: + return "_"; + case 96: + return "`"; + case 97: + return "a"; + case 98: + return "b"; + case 99: + return "c"; + case 100: + return "d"; + case 101: + return "e"; + case 102: + return "f"; + case 103: + return "g"; + case 104: + return "h"; + case 105: + return "i"; + case 106: + return "j"; + case 107: + return "k"; + case 108: + return "l"; + case 109: + return "m"; + case 110: + return "n"; + case 111: + return "o"; + case 112: + return "p"; + case 113: + return "q"; + case 114: + return "r"; + case 115: + return "s"; + case 116: + return "t"; + case 117: + return "u"; + case 118: + return "v"; + case 119: + return "w"; + case 120: + return "x"; + case 121: + return "y"; + case 122: + return "z"; + case 123: + return "{"; + case 124: + return "|"; + case 125: + return "}"; + case 126: + return "~"; + case 127: + return "\\x7F"; + case 128: + return "\\x80"; + case 129: + return "\\x81"; + case 130: + return "\\x82"; + case 131: + return "\\x83"; + case 132: + return "\\x84"; + case 133: + return "\\x85"; + case 134: + return "\\x86"; + case 135: + return "\\x87"; + case 136: + return "\\x88"; + case 137: + return "\\x89"; + case 138: + return "\\x8A"; + case 139: + return "\\x8B"; + case 140: + return "\\x8C"; + case 141: + return "\\x8D"; + case 142: + return "\\x8E"; + case 143: + return "\\x8F"; + case 144: + return "\\x90"; + case 145: + return "\\x91"; + case 146: + return "\\x92"; + case 147: + return "\\x93"; + case 148: + return "\\x94"; + case 149: + return "\\x95"; + case 150: + return "\\x96"; + case 151: + return "\\x97"; + case 152: + return "\\x98"; + case 153: + return "\\x99"; + case 154: + return "\\x9A"; + case 155: + return "\\x9B"; + case 156: + return "\\x9C"; + case 157: + return "\\x9D"; + case 158: + return "\\x9E"; + case 159: + return "\\x9F"; + case 160: + return "\\xA0"; + case 161: + return "\\xA1"; + case 162: + return "\\xA2"; + case 163: + return "\\xA3"; + case 164: + return "\\xA4"; + case 165: + return "\\xA5"; + case 166: + return "\\xA6"; + case 167: + return "\\xA7"; + case 168: + return "\\xA8"; + case 169: + return "\\xA9"; + case 170: + return "\\xAA"; + case 171: + return "\\xAB"; + case 172: + return "\\xAC"; + case 173: + return "\\xAD"; + case 174: + return "\\xAE"; + case 175: + return "\\xAF"; + case 176: + return "\\xB0"; + case 177: + return "\\xB1"; + case 178: + return "\\xB2"; + case 179: + return "\\xB3"; + case 180: + return "\\xB4"; + case 181: + return "\\xB5"; + case 182: + return "\\xB6"; + case 183: + return "\\xB7"; + case 184: + return "\\xB8"; + case 185: + return "\\xB9"; + case 186: + return "\\xBA"; + case 187: + return "\\xBB"; + case 188: + return "\\xBC"; + case 189: + return "\\xBD"; + case 190: + return "\\xBE"; + case 191: + return "\\xBF"; + case 192: + return "\\xC0"; + case 193: + return "\\xC1"; + case 194: + return "\\xC2"; + case 195: + return "\\xC3"; + case 196: + return "\\xC4"; + case 197: + return "\\xC5"; + case 198: + return "\\xC6"; + case 199: + return "\\xC7"; + case 200: + return "\\xC8"; + case 201: + return "\\xC9"; + case 202: + return "\\xCA"; + case 203: + return "\\xCB"; + case 204: + return "\\xCC"; + case 205: + return "\\xCD"; + case 206: + return "\\xCE"; + case 207: + return "\\xCF"; + case 208: + return "\\xD0"; + case 209: + return "\\xD1"; + case 210: + return "\\xD2"; + case 211: + return "\\xD3"; + case 212: + return "\\xD4"; + case 213: + return "\\xD5"; + case 214: + return "\\xD6"; + case 215: + return "\\xD7"; + case 216: + return "\\xD8"; + case 217: + return "\\xD9"; + case 218: + return "\\xDA"; + case 219: + return "\\xDB"; + case 220: + return "\\xDC"; + case 221: + return "\\xDD"; + case 222: + return "\\xDE"; + case 223: + return "\\xDF"; + case 224: + return "\\xE0"; + case 225: + return "\\xE1"; + case 226: + return "\\xE2"; + case 227: + return "\\xE3"; + case 228: + return "\\xE4"; + case 229: + return "\\xE5"; + case 230: + return "\\xE6"; + case 231: + return "\\xE7"; + case 232: + return "\\xE8"; + case 233: + return "\\xE9"; + case 234: + return "\\xEA"; + case 235: + return "\\xEB"; + case 236: + return "\\xEC"; + case 237: + return "\\xED"; + case 238: + return "\\xEE"; + case 239: + return "\\xEF"; + case 240: + return "\\xF0"; + case 241: + return "\\xF1"; + case 242: + return "\\xF2"; + case 243: + return "\\xF3"; + case 244: + return "\\xF4"; + case 245: + return "\\xF5"; + case 246: + return "\\xF6"; + case 247: + return "\\xF7"; + case 248: + return "\\xF8"; + case 249: + return "\\xF9"; + case 250: + return "\\xFA"; + case 251: + return "\\xFB"; + case 252: + return "\\xFC"; + case 253: + return "\\xFD"; + case 254: + return "\\xFE"; + case 255: + return "\\xFF"; + default: + assert(0); /* never gets here */ + return "dead code"; + } + assert(0); /* never gets here */ +} + +#endif /* XML_DTD */ + +static unsigned long +getDebugLevel(const char *variableName, unsigned long defaultDebugLevel) { + const char *const valueOrNull = getenv(variableName); + if (valueOrNull == NULL) { + return defaultDebugLevel; + } + const char *const value = valueOrNull; + + errno = 0; + char *afterValue = (char *)value; + unsigned long debugLevel = strtoul(value, &afterValue, 10); + if ((errno != 0) || (afterValue[0] != '\0')) { + errno = 0; + return defaultDebugLevel; + } + + return debugLevel; +} diff --git a/Modules/expat/xmlrole.c b/Modules/expat/xmlrole.c index 4d3e3e86e9e864..08173b0fd541d2 100644 --- a/Modules/expat/xmlrole.c +++ b/Modules/expat/xmlrole.c @@ -7,7 +7,14 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper <coopercc(a)users.sourceforge.net> + Copyright (c) 2002 Greg Stein <gstein(a)users.sourceforge.net> + Copyright (c) 2002-2006 Karl Waclawek <karl(a)waclawek.net> + Copyright (c) 2002-2003 Fred L. Drake, Jr. <fdrake(a)users.sourceforge.net> + Copyright (c) 2005-2009 Steven Solie <ssolie(a)users.sourceforge.net> + Copyright (c) 2016-2021 Sebastian Pipping <sebastian(a)pipping.org> + Copyright (c) 2017 Rhodri James <rhodri(a)wildebeest.org.uk> + Copyright (c) 2019 David Loffredo <loffredo(a)steptools.com> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -34,11 +41,9 @@ #ifdef _WIN32 # include "winconfig.h" -#else -# ifdef HAVE_EXPAT_CONFIG_H -# include <expat_config.h> -# endif -#endif /* ndef _WIN32 */ +#endif + +#include <expat_config.h> #include "expat_external.h" #include "internal.h" @@ -1220,6 +1225,8 @@ common(PROLOG_STATE *state, int tok) { #ifdef XML_DTD if (! state->documentEntity && tok == XML_TOK_PARAM_ENTITY_REF) return XML_ROLE_INNER_PARAM_ENTITY_REF; +#else + UNUSED_P(tok); #endif state->handler = error; return XML_ROLE_ERROR; diff --git a/Modules/expat/xmlrole.h b/Modules/expat/xmlrole.h index 036aba64fd29c6..d6e1fa150a108a 100644 --- a/Modules/expat/xmlrole.h +++ b/Modules/expat/xmlrole.h @@ -7,7 +7,10 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper <coopercc(a)users.sourceforge.net> + Copyright (c) 2002 Karl Waclawek <karl(a)waclawek.net> + Copyright (c) 2002 Fred L. Drake, Jr. <fdrake(a)users.sourceforge.net> + Copyright (c) 2017 Sebastian Pipping <sebastian(a)pipping.org> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining diff --git a/Modules/expat/xmltok.c b/Modules/expat/xmltok.c index 11e9d1ccdad423..f2b6b406067ea9 100644 --- a/Modules/expat/xmltok.c +++ b/Modules/expat/xmltok.c @@ -7,7 +7,19 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper <coopercc(a)users.sourceforge.net> + Copyright (c) 2001-2003 Fred L. Drake, Jr. <fdrake(a)users.sourceforge.net> + Copyright (c) 2002 Greg Stein <gstein(a)users.sourceforge.net> + Copyright (c) 2002-2016 Karl Waclawek <karl(a)waclawek.net> + Copyright (c) 2005-2009 Steven Solie <ssolie(a)users.sourceforge.net> + Copyright (c) 2016-2021 Sebastian Pipping <sebastian(a)pipping.org> + Copyright (c) 2016 Pascal Cuoq <cuoq(a)trust-in-soft.com> + Copyright (c) 2016 Don Lewis <truckman(a)apache.org> + Copyright (c) 2017 Rhodri James <rhodri(a)wildebeest.org.uk> + Copyright (c) 2017 Alexander Bluhm <alexander.bluhm(a)gmx.net> + Copyright (c) 2017 Benbuck Nason <bnason(a)netflix.com> + Copyright (c) 2017 José Gutiérrez de la Concha <jose(a)zeroc.com> + Copyright (c) 2019 David Loffredo <loffredo(a)steptools.com> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -32,23 +44,13 @@ #include <stddef.h> #include <string.h> /* memcpy */ - -#if defined(_MSC_VER) && (_MSC_VER <= 1700) -/* for vs2012/11.0/1700 and earlier Visual Studio compilers */ -# define bool int -# define false 0 -# define true 1 -#else -# include <stdbool.h> -#endif +#include <stdbool.h> #ifdef _WIN32 # include "winconfig.h" -#else -# ifdef HAVE_EXPAT_CONFIG_H -# include <expat_config.h> -# endif -#endif /* ndef _WIN32 */ +#endif + +#include <expat_config.h> #include "expat_external.h" #include "internal.h" @@ -269,8 +271,14 @@ sb_byteToAscii(const ENCODING *enc, const char *p) { #define IS_NAME_CHAR(enc, p, n) (AS_NORMAL_ENCODING(enc)->isName##n(enc, p)) #define IS_NMSTRT_CHAR(enc, p, n) (AS_NORMAL_ENCODING(enc)->isNmstrt##n(enc, p)) -#define IS_INVALID_CHAR(enc, p, n) \ - (AS_NORMAL_ENCODING(enc)->isInvalid##n(enc, p)) +#ifdef XML_MIN_SIZE +# define IS_INVALID_CHAR(enc, p, n) \ + (AS_NORMAL_ENCODING(enc)->isInvalid##n \ + && AS_NORMAL_ENCODING(enc)->isInvalid##n(enc, p)) +#else +# define IS_INVALID_CHAR(enc, p, n) \ + (AS_NORMAL_ENCODING(enc)->isInvalid##n(enc, p)) +#endif #ifdef XML_MIN_SIZE # define IS_NAME_CHAR_MINBPC(enc, p) \ @@ -589,13 +597,13 @@ static const struct normal_encoding ascii_encoding static int PTRFASTCALL unicode_byte_type(char hi, char lo) { switch ((unsigned char)hi) { - /* 0xD800–0xDBFF first 16-bit code unit or high surrogate (W1) */ + /* 0xD800-0xDBFF first 16-bit code unit or high surrogate (W1) */ case 0xD8: case 0xD9: case 0xDA: case 0xDB: return BT_LEAD4; - /* 0xDC00–0xDFFF second 16-bit code unit or low surrogate (W2) */ + /* 0xDC00-0xDFFF second 16-bit code unit or low surrogate (W2) */ case 0xDC: case 0xDD: case 0xDE: diff --git a/Modules/expat/xmltok.h b/Modules/expat/xmltok.h index 2adbf5307befae..6f630c2f9ba96d 100644 --- a/Modules/expat/xmltok.h +++ b/Modules/expat/xmltok.h @@ -7,7 +7,11 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper <coopercc(a)users.sourceforge.net> + Copyright (c) 2002 Fred L. Drake, Jr. <fdrake(a)users.sourceforge.net> + Copyright (c) 2002-2005 Karl Waclawek <karl(a)waclawek.net> + Copyright (c) 2016-2017 Sebastian Pipping <sebastian(a)pipping.org> + Copyright (c) 2017 Rhodri James <rhodri(a)wildebeest.org.uk> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining diff --git a/Modules/expat/xmltok_impl.c b/Modules/expat/xmltok_impl.c index c209221cd79d13..0430591b42636e 100644 --- a/Modules/expat/xmltok_impl.c +++ b/Modules/expat/xmltok_impl.c @@ -1,4 +1,4 @@ -/* This file is included! +/* This file is included (from xmltok.c, 1-3 times depending on XML_MIN_SIZE)! __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -7,7 +7,15 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper <coopercc(a)users.sourceforge.net> + Copyright (c) 2002 Fred L. Drake, Jr. <fdrake(a)users.sourceforge.net> + Copyright (c) 2002-2016 Karl Waclawek <karl(a)waclawek.net> + Copyright (c) 2016-2021 Sebastian Pipping <sebastian(a)pipping.org> + Copyright (c) 2017 Rhodri James <rhodri(a)wildebeest.org.uk> + Copyright (c) 2018 Benjamin Peterson <benjamin(a)python.org> + Copyright (c) 2018 Anton Maklakov <antmak.pub(a)gmail.com> + Copyright (c) 2019 David Loffredo <loffredo(a)steptools.com> + Copyright (c) 2020 Boris Kolpackov <boris(a)codesynthesis.com> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -32,7 +40,7 @@ #ifdef XML_TOK_IMPL_C -# ifndef IS_INVALID_CHAR +# ifndef IS_INVALID_CHAR // i.e. for UTF-16 and XML_MIN_SIZE not defined # define IS_INVALID_CHAR(enc, ptr, n) (0) # endif @@ -1768,13 +1776,14 @@ PREFIX(updatePosition)(const ENCODING *enc, const char *ptr, const char *end, # define LEAD_CASE(n) \ case BT_LEAD##n: \ ptr += n; \ + pos->columnNumber++; \ break; LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4) # undef LEAD_CASE case BT_LF: - pos->columnNumber = (XML_Size)-1; + pos->columnNumber = 0; pos->lineNumber++; ptr += MINBPC(enc); break; @@ -1783,13 +1792,13 @@ PREFIX(updatePosition)(const ENCODING *enc, const char *ptr, const char *end, ptr += MINBPC(enc); if (HAS_CHAR(enc, ptr, end) && BYTE_TYPE(enc, ptr) == BT_LF) ptr += MINBPC(enc); - pos->columnNumber = (XML_Size)-1; + pos->columnNumber = 0; break; default: ptr += MINBPC(enc); + pos->columnNumber++; break; } - pos->columnNumber++; } } diff --git a/Modules/expat/xmltok_impl.h b/Modules/expat/xmltok_impl.h index e925dbc7e2c833..c518aada013df2 100644 --- a/Modules/expat/xmltok_impl.h +++ b/Modules/expat/xmltok_impl.h @@ -7,7 +7,8 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper <coopercc(a)users.sourceforge.net> + Copyright (c) 2017-2019 Sebastian Pipping <sebastian(a)pipping.org> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining diff --git a/Modules/expat/xmltok_ns.c b/Modules/expat/xmltok_ns.c index 919c74e9f97fe8..5fd83922359400 100644 --- a/Modules/expat/xmltok_ns.c +++ b/Modules/expat/xmltok_ns.c @@ -7,7 +7,11 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper <coopercc(a)users.sourceforge.net> + Copyright (c) 2002 Greg Stein <gstein(a)users.sourceforge.net> + Copyright (c) 2002 Fred L. Drake, Jr. <fdrake(a)users.sourceforge.net> + Copyright (c) 2002-2006 Karl Waclawek <karl(a)waclawek.net> + Copyright (c) 2017 Sebastian Pipping <sebastian(a)pipping.org> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining

1 0

bpo-38965: Fix faulthandler._stack_overflow() on GCC 10 (GH-17467) (GH-28079)
by ned-deily Aug. 30, 2021

Aug. 30, 2021

https://github.com/python/cpython/commit/8934bb0c3179e4c020cd6f08dea64bccbf… commit: 8934bb0c3179e4c020cd6f08dea64bccbf56ffa2 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: ned-deily <nad(a)python.org> date: 2021-08-31T02:24:50-04:00 summary: bpo-38965: Fix faulthandler._stack_overflow() on GCC 10 (GH-17467) (GH-28079) Use the "volatile" keyword to prevent tail call optimization on any compiler, rather than relying on compiler specific pragma. (cherry picked from commit 8b787964e0a647caa0558b7c29ae501470d727d9) Co-authored-by: Victor Stinner <vstinner(a)python.org> (cherry picked from commit 5044c889dfced2f43e2cccb673d889a4882f6b3b) Co-authored-by: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> files: A Misc/NEWS.d/next/Tests/2019-12-04-17-08-55.bpo-38965.yqax3m.rst M Modules/faulthandler.c diff --git a/Misc/NEWS.d/next/Tests/2019-12-04-17-08-55.bpo-38965.yqax3m.rst b/Misc/NEWS.d/next/Tests/2019-12-04-17-08-55.bpo-38965.yqax3m.rst new file mode 100644 index 0000000000000..517a1371eacd9 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2019-12-04-17-08-55.bpo-38965.yqax3m.rst @@ -0,0 +1,3 @@ +Fix test_faulthandler on GCC 10. Use the "volatile" keyword in +``faulthandler._stack_overflow()`` to prevent tail call optimization on any +compiler, rather than relying on compiler specific pragma. diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 890c64577cec1..3e0a7b127d4c7 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -1091,18 +1091,14 @@ faulthandler_fatal_error_py(PyObject *self, PyObject *args) #if defined(HAVE_SIGALTSTACK) && defined(HAVE_SIGACTION) #define FAULTHANDLER_STACK_OVERFLOW -#ifdef __INTEL_COMPILER - /* Issue #23654: Turn off ICC's tail call optimization for the - * stack_overflow generator. ICC turns the recursive tail call into - * a loop. */ -# pragma intel optimization_level 0 -#endif -static -uintptr_t +static uintptr_t stack_overflow(uintptr_t min_sp, uintptr_t max_sp, size_t *depth) { - /* allocate 4096 bytes on the stack at each call */ - unsigned char buffer[4096]; + /* Allocate (at least) 4096 bytes on the stack at each call. + + bpo-23654, bpo-38965: use volatile keyword to prevent tail call + optimization. */ + volatile unsigned char buffer[4096]; uintptr_t sp = (uintptr_t)&buffer; *depth += 1; if (sp < min_sp || max_sp < sp)

1 0

[3.7] bpo-44394: Update libexpat copy to 2.4.1 (GH-26945) (GH-28042)
by ned-deily Aug. 30, 2021

Aug. 30, 2021

https://github.com/python/cpython/commit/79101b890ee021a901a8b6837a3a320d57… commit: 79101b890ee021a901a8b6837a3a320d57adb725 branch: 3.7 author: Łukasz Langa <lukasz(a)langa.pl> committer: ned-deily <nad(a)python.org> date: 2021-08-31T01:11:53-04:00 summary: [3.7] bpo-44394: Update libexpat copy to 2.4.1 (GH-26945) (GH-28042) Update the vendored copy of libexpat to 2.4.1 (from 2.2.8) to get the fix for the CVE-2013-0340 "Billion Laughs" vulnerability. This copy is most used on Windows and macOS. Co-authored-by: Victor Stinner <vstinner(a)python.org> Co-authored-by: Łukasz Langa <lukasz(a)langa.pl>. (cherry picked from commit 3fc5d84046ddbd66abac5b598956ea34605a4e5d) files: A Misc/NEWS.d/next/Security/2021-06-29-02-45-53.bpo-44394.A220N1.rst M Doc/library/xml.rst M Modules/expat/COPYING M Modules/expat/ascii.h M Modules/expat/asciitab.h M Modules/expat/expat.h M Modules/expat/expat_external.h M Modules/expat/iasciitab.h M Modules/expat/internal.h M Modules/expat/latin1tab.h M Modules/expat/nametab.h M Modules/expat/siphash.h M Modules/expat/utf8tab.h M Modules/expat/winconfig.h M Modules/expat/xmlparse.c M Modules/expat/xmlrole.c M Modules/expat/xmlrole.h M Modules/expat/xmltok.c M Modules/expat/xmltok.h M Modules/expat/xmltok_impl.c M Modules/expat/xmltok_impl.h M Modules/expat/xmltok_ns.c diff --git a/Doc/library/xml.rst b/Doc/library/xml.rst index fb86b6f5564d76..01205cbaf45bf8 100644 --- a/Doc/library/xml.rst +++ b/Doc/library/xml.rst @@ -60,22 +60,26 @@ circumvent firewalls. The following table gives an overview of the known attacks and whether the various modules are vulnerable to them. -========================= ============== =============== ============== ============== ============== -kind sax etree minidom pulldom xmlrpc -========================= ============== =============== ============== ============== ============== -billion laughs **Vulnerable** **Vulnerable** **Vulnerable** **Vulnerable** **Vulnerable** -quadratic blowup **Vulnerable** **Vulnerable** **Vulnerable** **Vulnerable** **Vulnerable** -external entity expansion Safe (4) Safe (1) Safe (2) Safe (4) Safe (3) -`DTD`_ retrieval Safe (4) Safe Safe Safe (4) Safe -decompression bomb Safe Safe Safe Safe **Vulnerable** -========================= ============== =============== ============== ============== ============== - -1. :mod:`xml.etree.ElementTree` doesn't expand external entities and raises a +========================= ================== ================== ================== ================== ================== +kind sax etree minidom pulldom xmlrpc +========================= ================== ================== ================== ================== ================== +billion laughs **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) +quadratic blowup **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) +external entity expansion Safe (5) Safe (2) Safe (3) Safe (5) Safe (4) +`DTD`_ retrieval Safe (5) Safe Safe Safe (5) Safe +decompression bomb Safe Safe Safe Safe **Vulnerable** +========================= ================== ================== ================== ================== ================== + +1. Expat 2.4.1 and newer is not vulnerable to the "billion laughs" and + "quadratic blowup" vulnerabilities. Items still listed as vulnerable due to + potential reliance on system-provided libraries. Check + :data:`pyexpat.EXPAT_VERSION`. +2. :mod:`xml.etree.ElementTree` doesn't expand external entities and raises a :exc:`ParserError` when an entity occurs. -2. :mod:`xml.dom.minidom` doesn't expand external entities and simply returns +3. :mod:`xml.dom.minidom` doesn't expand external entities and simply returns the unexpanded entity verbatim. -3. :mod:`xmlrpclib` doesn't expand external entities and omits them. -4. Since Python 3.7.1, external general entities are no longer processed by +4. :mod:`xmlrpclib` doesn't expand external entities and omits them. +5. Since Python 3.7.1, external general entities are no longer processed by default. diff --git a/Misc/NEWS.d/next/Security/2021-06-29-02-45-53.bpo-44394.A220N1.rst b/Misc/NEWS.d/next/Security/2021-06-29-02-45-53.bpo-44394.A220N1.rst new file mode 100644 index 00000000000000..e32563d2535c7e --- /dev/null +++ b/Misc/NEWS.d/next/Security/2021-06-29-02-45-53.bpo-44394.A220N1.rst @@ -0,0 +1,3 @@ +Update the vendored copy of libexpat to 2.4.1 (from 2.2.8) to get the fix +for the CVE-2013-0340 "Billion Laughs" vulnerability. This copy is most used +on Windows and macOS. diff --git a/Modules/expat/COPYING b/Modules/expat/COPYING index 8d288f0f28fddd..3c0142e71c8d4f 100644 --- a/Modules/expat/COPYING +++ b/Modules/expat/COPYING @@ -1,5 +1,5 @@ Copyright (c) 1998-2000 Thai Open Source Software Center Ltd and Clark Cooper -Copyright (c) 2001-2017 Expat maintainers +Copyright (c) 2001-2019 Expat maintainers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/Modules/expat/ascii.h b/Modules/expat/ascii.h index c3587e57332bff..1f594d2e54b4d2 100644 --- a/Modules/expat/ascii.h +++ b/Modules/expat/ascii.h @@ -6,8 +6,11 @@ \___/_/\_\ .__/ \__,_|\__| |_| XML parser - Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 1999-2000 Thai Open Source Software Center Ltd + Copyright (c) 2000 Clark Cooper <coopercc(a)users.sourceforge.net> + Copyright (c) 2002 Fred L. Drake, Jr. <fdrake(a)users.sourceforge.net> + Copyright (c) 2007 Karl Waclawek <karl(a)waclawek.net> + Copyright (c) 2017 Sebastian Pipping <sebastian(a)pipping.org> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining diff --git a/Modules/expat/asciitab.h b/Modules/expat/asciitab.h index 63b1d1b4482efa..af766fb24785ea 100644 --- a/Modules/expat/asciitab.h +++ b/Modules/expat/asciitab.h @@ -7,7 +7,9 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper <coopercc(a)users.sourceforge.net> + Copyright (c) 2002 Fred L. Drake, Jr. <fdrake(a)users.sourceforge.net> + Copyright (c) 2017 Sebastian Pipping <sebastian(a)pipping.org> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index 6c8eb1fda4a30c..b7d6d354801b3d 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -7,7 +7,14 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper <coopercc(a)users.sourceforge.net> + Copyright (c) 2000-2005 Fred L. Drake, Jr. <fdrake(a)users.sourceforge.net> + Copyright (c) 2001-2002 Greg Stein <gstein(a)users.sourceforge.net> + Copyright (c) 2002-2016 Karl Waclawek <karl(a)waclawek.net> + Copyright (c) 2016-2021 Sebastian Pipping <sebastian(a)pipping.org> + Copyright (c) 2016 Cristian Rodríguez <crrodriguez(a)opensuse.org> + Copyright (c) 2016 Thomas Beutlich <tc(a)tbeu.de> + Copyright (c) 2017 Rhodri James <rhodri(a)wildebeest.org.uk> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -115,7 +122,11 @@ enum XML_Error { XML_ERROR_RESERVED_PREFIX_XMLNS, XML_ERROR_RESERVED_NAMESPACE_URI, /* Added in 2.2.1. */ - XML_ERROR_INVALID_ARGUMENT + XML_ERROR_INVALID_ARGUMENT, + /* Added in 2.3.0. */ + XML_ERROR_NO_BUFFER, + /* Added in 2.4.0. */ + XML_ERROR_AMPLIFICATION_LIMIT_BREACH }; enum XML_Content_Type { @@ -318,7 +329,7 @@ typedef void(XMLCALL *XML_EndDoctypeDeclHandler)(void *userData); For internal entities (<!ENTITY foo "bar">), value will be non-NULL and systemId, publicID, and notationName will be NULL. - The value string is NOT nul-terminated; the length is provided in + The value string is NOT null-terminated; the length is provided in the value_length argument. Since it is legal to have zero-length values, do not use this argument to test for internal entities. @@ -513,7 +524,7 @@ typedef struct { Otherwise it must return XML_STATUS_ERROR. If info does not describe a suitable encoding, then the parser will - return an XML_UNKNOWN_ENCODING error. + return an XML_ERROR_UNKNOWN_ENCODING error. */ typedef int(XMLCALL *XML_UnknownEncodingHandler)(void *encodingHandlerData, const XML_Char *name, @@ -707,7 +718,7 @@ XML_GetBase(XML_Parser parser); /* Returns the number of the attribute/value pairs passed in last call to the XML_StartElementHandler that were specified in the start-tag rather than defaulted. Each attribute/value pair counts as 2; thus - this correspondds to an index into the atts array passed to the + this corresponds to an index into the atts array passed to the XML_StartElementHandler. Returns -1 if parser == NULL. */ XMLPARSEAPI(int) @@ -716,7 +727,7 @@ XML_GetSpecifiedAttributeCount(XML_Parser parser); /* Returns the index of the ID attribute passed in the last call to XML_StartElementHandler, or -1 if there is no ID attribute or parser == NULL. Each attribute/value pair counts as 2; thus this - correspondds to an index into the atts array passed to the + corresponds to an index into the atts array passed to the XML_StartElementHandler. */ XMLPARSEAPI(int) @@ -997,7 +1008,10 @@ enum XML_FeatureEnum { XML_FEATURE_SIZEOF_XML_LCHAR, XML_FEATURE_NS, XML_FEATURE_LARGE_SIZE, - XML_FEATURE_ATTR_INFO + XML_FEATURE_ATTR_INFO, + /* Added in Expat 2.4.0. */ + XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT, + XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT /* Additional features must be added to the end of this enum. */ }; @@ -1010,12 +1024,24 @@ typedef struct { XMLPARSEAPI(const XML_Feature *) XML_GetFeatureList(void); +#ifdef XML_DTD +/* Added in Expat 2.4.0. */ +XMLPARSEAPI(XML_Bool) +XML_SetBillionLaughsAttackProtectionMaximumAmplification( + XML_Parser parser, float maximumAmplificationFactor); + +/* Added in Expat 2.4.0. */ +XMLPARSEAPI(XML_Bool) +XML_SetBillionLaughsAttackProtectionActivationThreshold( + XML_Parser parser, unsigned long long activationThresholdBytes); +#endif + /* Expat follows the semantic versioning convention. See http://semver.org. */ #define XML_MAJOR_VERSION 2 -#define XML_MINOR_VERSION 2 -#define XML_MICRO_VERSION 8 +#define XML_MINOR_VERSION 4 +#define XML_MICRO_VERSION 1 #ifdef __cplusplus } diff --git a/Modules/expat/expat_external.h b/Modules/expat/expat_external.h index f2b75dda8e2798..12c560e14716ff 100644 --- a/Modules/expat/expat_external.h +++ b/Modules/expat/expat_external.h @@ -7,7 +7,14 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper <coopercc(a)users.sourceforge.net> + Copyright (c) 2000-2004 Fred L. Drake, Jr. <fdrake(a)users.sourceforge.net> + Copyright (c) 2001-2002 Greg Stein <gstein(a)users.sourceforge.net> + Copyright (c) 2002-2006 Karl Waclawek <karl(a)waclawek.net> + Copyright (c) 2016 Cristian Rodríguez <crrodriguez(a)opensuse.org> + Copyright (c) 2016-2019 Sebastian Pipping <sebastian(a)pipping.org> + Copyright (c) 2017 Rhodri James <rhodri(a)wildebeest.org.uk> + Copyright (c) 2018 Yury Gribov <tetra2005(a)gmail.com> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining diff --git a/Modules/expat/iasciitab.h b/Modules/expat/iasciitab.h index ea97cfcf678e06..5d8646f2a318b8 100644 --- a/Modules/expat/iasciitab.h +++ b/Modules/expat/iasciitab.h @@ -7,7 +7,9 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper <coopercc(a)users.sourceforge.net> + Copyright (c) 2002 Fred L. Drake, Jr. <fdrake(a)users.sourceforge.net> + Copyright (c) 2017 Sebastian Pipping <sebastian(a)pipping.org> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining diff --git a/Modules/expat/internal.h b/Modules/expat/internal.h index 60913dab762f8f..444eba0fb03170 100644 --- a/Modules/expat/internal.h +++ b/Modules/expat/internal.h @@ -25,8 +25,12 @@ \___/_/\_\ .__/ \__,_|\__| |_| XML parser - Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2002-2003 Fred L. Drake, Jr. <fdrake(a)users.sourceforge.net> + Copyright (c) 2002-2006 Karl Waclawek <karl(a)waclawek.net> + Copyright (c) 2003 Greg Stein <gstein(a)users.sourceforge.net> + Copyright (c) 2016-2021 Sebastian Pipping <sebastian(a)pipping.org> + Copyright (c) 2018 Yury Gribov <tetra2005(a)gmail.com> + Copyright (c) 2019 David Loffredo <loffredo(a)steptools.com> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -101,22 +105,58 @@ # endif #endif +#include <limits.h> // ULONG_MAX + +#if defined(_WIN32) && ! defined(__USE_MINGW_ANSI_STDIO) +# define EXPAT_FMT_ULL(midpart) "%" midpart "I64u" +# if defined(_WIN64) // Note: modifiers "td" and "zu" do not work for MinGW +# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "I64d" +# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "I64u" +# else +# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "d" +# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "u" +# endif +#else +# define EXPAT_FMT_ULL(midpart) "%" midpart "llu" +# if ! defined(ULONG_MAX) +# error Compiler did not define ULONG_MAX for us +# elif ULONG_MAX == 18446744073709551615u // 2^64-1 +# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "ld" +# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "lu" +# else +# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "d" +# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "u" +# endif +#endif + #ifndef UNUSED_P # define UNUSED_P(p) (void)p #endif +/* NOTE BEGIN If you ever patch these defaults to greater values + for non-attack XML payload in your environment, + please file a bug report with libexpat. Thank you! +*/ +#define EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT \ + 100.0f +#define EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT \ + 8388608 // 8 MiB, 2^23 +/* NOTE END */ + +#include "expat.h" // so we can use type XML_Parser below + #ifdef __cplusplus extern "C" { #endif -#ifdef XML_ENABLE_VISIBILITY -# if XML_ENABLE_VISIBILITY -__attribute__((visibility("default"))) -# endif +void _INTERNAL_trim_to_complete_utf8_characters(const char *from, + const char **fromLimRef); + +#if defined(XML_DTD) +unsigned long long testingAccountingGetCountBytesDirect(XML_Parser parser); +unsigned long long testingAccountingGetCountBytesIndirect(XML_Parser parser); +const char *unsignedCharToPrintable(unsigned char c); #endif -void -_INTERNAL_trim_to_complete_utf8_characters(const char *from, - const char **fromLimRef); #ifdef __cplusplus } diff --git a/Modules/expat/latin1tab.h b/Modules/expat/latin1tab.h index 6f916041355a86..b681d278af6569 100644 --- a/Modules/expat/latin1tab.h +++ b/Modules/expat/latin1tab.h @@ -7,7 +7,9 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper <coopercc(a)users.sourceforge.net> + Copyright (c) 2002 Fred L. Drake, Jr. <fdrake(a)users.sourceforge.net> + Copyright (c) 2017 Sebastian Pipping <sebastian(a)pipping.org> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining diff --git a/Modules/expat/nametab.h b/Modules/expat/nametab.h index 3681df348eebd6..63485446b96727 100644 --- a/Modules/expat/nametab.h +++ b/Modules/expat/nametab.h @@ -6,8 +6,8 @@ \___/_/\_\ .__/ \__,_|\__| |_| XML parser - Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper <coopercc(a)users.sourceforge.net> + Copyright (c) 2017 Sebastian Pipping <sebastian(a)pipping.org> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining diff --git a/Modules/expat/siphash.h b/Modules/expat/siphash.h index bfee65a332f1bf..e5406d7ee9eb54 100644 --- a/Modules/expat/siphash.h +++ b/Modules/expat/siphash.h @@ -11,6 +11,9 @@ * -------------------------------------------------------------------------- * HISTORY: * + * 2020-10-03 (Sebastian Pipping) + * - Drop support for Visual Studio 9.0/2008 and earlier + * * 2019-08-03 (Sebastian Pipping) * - Mark part of sip24_valid as to be excluded from clang-format * - Re-format code using clang-format 9 @@ -96,15 +99,7 @@ #define SIPHASH_H #include <stddef.h> /* size_t */ - -#if defined(_WIN32) && defined(_MSC_VER) && (_MSC_VER < 1600) -/* For vs2003/7.1 up to vs2008/9.0; _MSC_VER 1600 is vs2010/10.0 */ -typedef unsigned __int8 uint8_t; -typedef unsigned __int32 uint32_t; -typedef unsigned __int64 uint64_t; -#else -# include <stdint.h> /* uint64_t uint32_t uint8_t */ -#endif +#include <stdint.h> /* uint64_t uint32_t uint8_t */ /* * Workaround to not require a C++11 compiler for using ULL suffix diff --git a/Modules/expat/utf8tab.h b/Modules/expat/utf8tab.h index a22986acbb9526..88efcf91cc16a6 100644 --- a/Modules/expat/utf8tab.h +++ b/Modules/expat/utf8tab.h @@ -7,7 +7,9 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper <coopercc(a)users.sourceforge.net> + Copyright (c) 2002 Fred L. Drake, Jr. <fdrake(a)users.sourceforge.net> + Copyright (c) 2017 Sebastian Pipping <sebastian(a)pipping.org> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining diff --git a/Modules/expat/winconfig.h b/Modules/expat/winconfig.h index 562a4a82dc1d63..2ecd61b5b94820 100644 --- a/Modules/expat/winconfig.h +++ b/Modules/expat/winconfig.h @@ -6,8 +6,10 @@ \___/_/\_\ .__/ \__,_|\__| |_| XML parser - Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper <coopercc(a)users.sourceforge.net> + Copyright (c) 2002 Greg Stein <gstein(a)users.sourceforge.net> + Copyright (c) 2005 Karl Waclawek <karl(a)waclawek.net> + Copyright (c) 2017-2021 Sebastian Pipping <sebastian(a)pipping.org> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -40,17 +42,4 @@ #include <memory.h> #include <string.h> -#if defined(HAVE_EXPAT_CONFIG_H) /* e.g. MinGW */ -# include <expat_config.h> -#else /* !defined(HAVE_EXPAT_CONFIG_H) */ - -# define XML_NS 1 -# define XML_DTD 1 -# define XML_CONTEXT_BYTES 1024 - -/* we will assume all Windows platforms are little endian */ -# define BYTEORDER 1234 - -#endif /* !defined(HAVE_EXPAT_CONFIG_H) */ - #endif /* ndef WINCONFIG_H */ diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index e740f0e19c7d4a..5ba56eaea6357a 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -1,4 +1,4 @@ -/* f2d0ab6d1d4422a08cf1cf3bbdfba96b49dea42fb5ff4615e03a2a25c306e769 (2.2.8+) +/* 8539b9040d9d901366a62560a064af7cb99811335784b363abc039c5b0ebc416 (2.4.1+) __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -7,7 +7,31 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper <coopercc(a)users.sourceforge.net> + Copyright (c) 2000-2006 Fred L. Drake, Jr. <fdrake(a)users.sourceforge.net> + Copyright (c) 2001-2002 Greg Stein <gstein(a)users.sourceforge.net> + Copyright (c) 2002-2016 Karl Waclawek <karl(a)waclawek.net> + Copyright (c) 2005-2009 Steven Solie <ssolie(a)users.sourceforge.net> + Copyright (c) 2016 Eric Rahm <erahm(a)mozilla.com> + Copyright (c) 2016-2021 Sebastian Pipping <sebastian(a)pipping.org> + Copyright (c) 2016 Gaurav <g.gupta(a)samsung.com> + Copyright (c) 2016 Thomas Beutlich <tc(a)tbeu.de> + Copyright (c) 2016 Gustavo Grieco <gustavo.grieco(a)imag.fr> + Copyright (c) 2016 Pascal Cuoq <cuoq(a)trust-in-soft.com> + Copyright (c) 2016 Ed Schouten <ed(a)nuxi.nl> + Copyright (c) 2017-2018 Rhodri James <rhodri(a)wildebeest.org.uk> + Copyright (c) 2017 Václav Slavík <vaclav(a)slavik.io> + Copyright (c) 2017 Viktor Szakats <commit(a)vsz.me> + Copyright (c) 2017 Chanho Park <chanho61.park(a)samsung.com> + Copyright (c) 2017 Rolf Eike Beer <eike(a)sf-mail.de> + Copyright (c) 2017 Hans Wennborg <hans(a)chromium.org> + Copyright (c) 2018 Anton Maklakov <antmak.pub(a)gmail.com> + Copyright (c) 2018 Benjamin Peterson <benjamin(a)python.org> + Copyright (c) 2018 Marco Maggi <marco.maggi-ipsu(a)poste.it> + Copyright (c) 2018 Mariusz Zaborski <oshogbo(a)vexillium.org> + Copyright (c) 2019 David Loffredo <loffredo(a)steptools.com> + Copyright (c) 2019-2020 Ben Wagner <bungeman(a)chromium.org> + Copyright (c) 2019 Vadim Zeitlin <vadim(a)zeitlins.org> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -36,7 +60,9 @@ #ifdef _WIN32 /* force stdlib to define rand_s() */ -# define _CRT_RAND_S +# if ! defined(_CRT_RAND_S) +# define _CRT_RAND_S +# endif #endif #include <stddef.h> @@ -45,6 +71,8 @@ #include <limits.h> /* UINT_MAX */ #include <stdio.h> /* fprintf */ #include <stdlib.h> /* getenv, rand_s */ +#include <stdint.h> /* uintptr_t */ +#include <math.h> /* isnan */ #ifdef _WIN32 # define getpid GetCurrentProcessId @@ -60,9 +88,9 @@ #ifdef _WIN32 # include "winconfig.h" -#elif defined(HAVE_EXPAT_CONFIG_H) -# include <expat_config.h> -#endif /* ndef _WIN32 */ +#endif + +#include <expat_config.h> #include "ascii.h" #include "expat.h" @@ -97,14 +125,14 @@ enabled. For end user security, that is probably not what you want. \ \ Your options include: \ - * Linux + glibc >=2.25 (getrandom): HAVE_GETRANDOM, \ - * Linux + glibc <2.25 (syscall SYS_getrandom): HAVE_SYSCALL_GETRANDOM, \ + * Linux >=3.17 + glibc >=2.25 (getrandom): HAVE_GETRANDOM, \ + * Linux >=3.17 + glibc (including <2.25) (syscall SYS_getrandom): HAVE_SYSCALL_GETRANDOM, \ * BSD / macOS >=10.7 (arc4random_buf): HAVE_ARC4RANDOM_BUF, \ - * BSD / macOS <10.7 (arc4random): HAVE_ARC4RANDOM, \ + * BSD / macOS (including <10.7) (arc4random): HAVE_ARC4RANDOM, \ * libbsd (arc4random_buf): HAVE_ARC4RANDOM_BUF + HAVE_LIBBSD, \ * libbsd (arc4random): HAVE_ARC4RANDOM + HAVE_LIBBSD, \ - * Linux / BSD / macOS (/dev/urandom): XML_DEV_URANDOM \ - * Windows (rand_s): _WIN32. \ + * Linux (including <3.17) / BSD / macOS (including <10.7) (/dev/urandom): XML_DEV_URANDOM, \ + * Windows >=Vista (rand_s): _WIN32. \ \ If insist on not using any of these, bypass this error by defining \ XML_POOR_ENTROPY; you have been warned. \ @@ -119,9 +147,7 @@ # define XmlGetInternalEncoding XmlGetUtf16InternalEncoding # define XmlGetInternalEncodingNS XmlGetUtf16InternalEncodingNS # define XmlEncode XmlUtf16Encode -/* Using pointer subtraction to convert to integer type. */ -# define MUST_CONVERT(enc, s) \ - (! (enc)->isUtf16 || (((char *)(s) - (char *)NULL) & 1)) +# define MUST_CONVERT(enc, s) (! (enc)->isUtf16 || (((uintptr_t)(s)) & 1)) typedef unsigned short ICHAR; #else # define XML_ENCODE_MAX XML_UTF8_ENCODE_MAX @@ -371,6 +397,31 @@ typedef struct open_internal_entity { XML_Bool betweenDecl; /* WFC: PE Between Declarations */ } OPEN_INTERNAL_ENTITY; +enum XML_Account { + XML_ACCOUNT_DIRECT, /* bytes directly passed to the Expat parser */ + XML_ACCOUNT_ENTITY_EXPANSION, /* intermediate bytes produced during entity + expansion */ + XML_ACCOUNT_NONE /* i.e. do not account, was accounted already */ +}; + +#ifdef XML_DTD +typedef unsigned long long XmlBigCount; +typedef struct accounting { + XmlBigCount countBytesDirect; + XmlBigCount countBytesIndirect; + int debugLevel; + float maximumAmplificationFactor; // >=1.0 + unsigned long long activationThresholdBytes; +} ACCOUNTING; + +typedef struct entity_stats { + unsigned int countEverOpened; + unsigned int currentDepth; + unsigned int maximumDepthSeen; + int debugLevel; +} ENTITY_STATS; +#endif /* XML_DTD */ + typedef enum XML_Error PTRCALL Processor(XML_Parser parser, const char *start, const char *end, const char **endPtr); @@ -401,16 +452,18 @@ static enum XML_Error initializeEncoding(XML_Parser parser); static enum XML_Error doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, int tok, const char *next, const char **nextPtr, - XML_Bool haveMore, XML_Bool allowClosingDoctype); + XML_Bool haveMore, XML_Bool allowClosingDoctype, + enum XML_Account account); static enum XML_Error processInternalEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl); static enum XML_Error doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, const char *start, const char *end, const char **endPtr, - XML_Bool haveMore); + XML_Bool haveMore, enum XML_Account account); static enum XML_Error doCdataSection(XML_Parser parser, const ENCODING *, const char **startPtr, const char *end, - const char **nextPtr, XML_Bool haveMore); + const char **nextPtr, XML_Bool haveMore, + enum XML_Account account); #ifdef XML_DTD static enum XML_Error doIgnoreSection(XML_Parser parser, const ENCODING *, const char **startPtr, const char *end, @@ -420,7 +473,8 @@ static enum XML_Error doIgnoreSection(XML_Parser parser, const ENCODING *, static void freeBindings(XML_Parser parser, BINDING *bindings); static enum XML_Error storeAtts(XML_Parser parser, const ENCODING *, const char *s, TAG_NAME *tagNamePtr, - BINDING **bindingsPtr); + BINDING **bindingsPtr, + enum XML_Account account); static enum XML_Error addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, const XML_Char *uri, BINDING **bindingsPtr); @@ -429,15 +483,18 @@ static int defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *, XML_Bool isCdata, XML_Parser parser); static enum XML_Error storeAttributeValue(XML_Parser parser, const ENCODING *, XML_Bool isCdata, const char *, - const char *, STRING_POOL *); + const char *, STRING_POOL *, + enum XML_Account account); static enum XML_Error appendAttributeValue(XML_Parser parser, const ENCODING *, XML_Bool isCdata, const char *, - const char *, STRING_POOL *); + const char *, STRING_POOL *, + enum XML_Account account); static ATTRIBUTE_ID *getAttributeId(XML_Parser parser, const ENCODING *enc, const char *start, const char *end); static int setElementTypePrefix(XML_Parser parser, ELEMENT_TYPE *); static enum XML_Error storeEntityValue(XML_Parser parser, const ENCODING *enc, - const char *start, const char *end); + const char *start, const char *end, + enum XML_Account account); static int reportProcessingInstruction(XML_Parser parser, const ENCODING *enc, const char *start, const char *end); static int reportComment(XML_Parser parser, const ENCODING *enc, @@ -501,6 +558,34 @@ static XML_Parser parserCreate(const XML_Char *encodingName, static void parserInit(XML_Parser parser, const XML_Char *encodingName); +#ifdef XML_DTD +static float accountingGetCurrentAmplification(XML_Parser rootParser); +static void accountingReportStats(XML_Parser originParser, const char *epilog); +static void accountingOnAbort(XML_Parser originParser); +static void accountingReportDiff(XML_Parser rootParser, + unsigned int levelsAwayFromRootParser, + const char *before, const char *after, + ptrdiff_t bytesMore, int source_line, + enum XML_Account account); +static XML_Bool accountingDiffTolerated(XML_Parser originParser, int tok, + const char *before, const char *after, + int source_line, + enum XML_Account account); + +static void entityTrackingReportStats(XML_Parser parser, ENTITY *entity, + const char *action, int sourceLine); +static void entityTrackingOnOpen(XML_Parser parser, ENTITY *entity, + int sourceLine); +static void entityTrackingOnClose(XML_Parser parser, ENTITY *entity, + int sourceLine); + +static XML_Parser getRootParserOf(XML_Parser parser, + unsigned int *outLevelDiff); +#endif /* XML_DTD */ + +static unsigned long getDebugLevel(const char *variableName, + unsigned long defaultDebugLevel); + #define poolStart(pool) ((pool)->start) #define poolEnd(pool) ((pool)->ptr) #define poolLength(pool) ((pool)->ptr - (pool)->start) @@ -614,6 +699,10 @@ struct XML_ParserStruct { enum XML_ParamEntityParsing m_paramEntityParsing; #endif unsigned long m_hash_secret_salt; +#ifdef XML_DTD + ACCOUNTING m_accounting; + ENTITY_STATS m_entity_stats; +#endif }; #define MALLOC(parser, s) (parser->m_mem.malloc_fcn((s))) @@ -734,6 +823,15 @@ writeRandomBytes_arc4random(void *target, size_t count) { #ifdef _WIN32 +/* Provide declaration of rand_s() for MinGW-32 (not 64, which has it), + as it didn't declare it in its header prior to version 5.3.0 of its + runtime package (mingwrt, containing stdlib.h). The upstream fix + was introduced at https://osdn.net/projects/mingw/ticket/39658 . */ +# if defined(__MINGW32__) && defined(__MINGW32_VERSION) \ + && __MINGW32_VERSION < 5003000L && ! defined(__MINGW64_VERSION_MAJOR) +__declspec(dllimport) int rand_s(unsigned int *); +# endif + /* Obtain entropy on Windows using the rand_s() function which * generates cryptographically secure random numbers. Internally it * uses RtlGenRandom API which is present in Windows XP and later. @@ -789,9 +887,8 @@ gather_time_entropy(void) { static unsigned long ENTROPY_DEBUG(const char *label, unsigned long entropy) { - const char *const EXPAT_ENTROPY_DEBUG = getenv("EXPAT_ENTROPY_DEBUG"); - if (EXPAT_ENTROPY_DEBUG && ! strcmp(EXPAT_ENTROPY_DEBUG, "1")) { - fprintf(stderr, "Entropy: %s --> 0x%0*lx (%lu bytes)\n", label, + if (getDebugLevel("EXPAT_ENTROPY_DEBUG", 0) >= 1u) { + fprintf(stderr, "expat: Entropy: %s --> 0x%0*lx (%lu bytes)\n", label, (int)sizeof(entropy) * 2, entropy, (unsigned long)sizeof(entropy)); } return entropy; @@ -1053,6 +1150,18 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { parser->m_paramEntityParsing = XML_PARAM_ENTITY_PARSING_NEVER; #endif parser->m_hash_secret_salt = 0; + +#ifdef XML_DTD + memset(&parser->m_accounting, 0, sizeof(ACCOUNTING)); + parser->m_accounting.debugLevel = getDebugLevel("EXPAT_ACCOUNTING_DEBUG", 0u); + parser->m_accounting.maximumAmplificationFactor + = EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT; + parser->m_accounting.activationThresholdBytes + = EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT; + + memset(&parser->m_entity_stats, 0, sizeof(ENTITY_STATS)); + parser->m_entity_stats.debugLevel = getDebugLevel("EXPAT_ENTITY_DEBUG", 0u); +#endif } /* moves list of bindings to m_freeBindingList */ @@ -1399,6 +1508,7 @@ XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD) { parser->m_useForeignDTD = useDTD; return XML_ERROR_NONE; #else + UNUSED_P(useDTD); return XML_ERROR_FEATURE_REQUIRES_XML_DTD; #endif } @@ -1780,7 +1890,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { int nLeftOver; enum XML_Status result; /* Detect overflow (a+b > MAX <==> b > MAX-a) */ - if (len > ((XML_Size)-1) / 2 - parser->m_parseEndByteIndex) { + if ((XML_Size)len > ((XML_Size)-1) / 2 - parser->m_parseEndByteIndex) { parser->m_errorCode = XML_ERROR_NO_MEMORY; parser->m_eventPtr = parser->m_eventEndPtr = NULL; parser->m_processor = errorProcessor; @@ -1872,6 +1982,12 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { parser->m_errorCode = XML_ERROR_FINISHED; return XML_STATUS_ERROR; case XML_INITIALIZED: + /* Has someone called XML_GetBuffer successfully before? */ + if (! parser->m_bufferPtr) { + parser->m_errorCode = XML_ERROR_NO_BUFFER; + return XML_STATUS_ERROR; + } + if (parser->m_parentParser == NULL && ! startParsing(parser)) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return XML_STATUS_ERROR; @@ -2155,7 +2271,7 @@ XML_GetInputContext(XML_Parser parser, int *offset, int *size) { (void)offset; (void)size; #endif /* defined XML_CONTEXT_BYTES */ - return (char *)0; + return (const char *)0; } XML_Size XMLCALL @@ -2316,6 +2432,14 @@ XML_ErrorString(enum XML_Error code) { /* Added in 2.2.5. */ case XML_ERROR_INVALID_ARGUMENT: /* Constant added in 2.2.1, already */ return XML_L("invalid argument"); + /* Added in 2.3.0. */ + case XML_ERROR_NO_BUFFER: + return XML_L( + "a successful prior call to function XML_GetBuffer is required"); + /* Added in 2.4.0. */ + case XML_ERROR_AMPLIFICATION_LIMIT_BREACH: + return XML_L( + "limit on input amplification factor (from DTD and entities) breached"); } return NULL; } @@ -2352,41 +2476,75 @@ XML_ExpatVersionInfo(void) { const XML_Feature *XMLCALL XML_GetFeatureList(void) { - static const XML_Feature features[] - = {{XML_FEATURE_SIZEOF_XML_CHAR, XML_L("sizeof(XML_Char)"), - sizeof(XML_Char)}, - {XML_FEATURE_SIZEOF_XML_LCHAR, XML_L("sizeof(XML_LChar)"), - sizeof(XML_LChar)}, + static const XML_Feature features[] = { + {XML_FEATURE_SIZEOF_XML_CHAR, XML_L("sizeof(XML_Char)"), + sizeof(XML_Char)}, + {XML_FEATURE_SIZEOF_XML_LCHAR, XML_L("sizeof(XML_LChar)"), + sizeof(XML_LChar)}, #ifdef XML_UNICODE - {XML_FEATURE_UNICODE, XML_L("XML_UNICODE"), 0}, + {XML_FEATURE_UNICODE, XML_L("XML_UNICODE"), 0}, #endif #ifdef XML_UNICODE_WCHAR_T - {XML_FEATURE_UNICODE_WCHAR_T, XML_L("XML_UNICODE_WCHAR_T"), 0}, + {XML_FEATURE_UNICODE_WCHAR_T, XML_L("XML_UNICODE_WCHAR_T"), 0}, #endif #ifdef XML_DTD - {XML_FEATURE_DTD, XML_L("XML_DTD"), 0}, + {XML_FEATURE_DTD, XML_L("XML_DTD"), 0}, #endif #ifdef XML_CONTEXT_BYTES - {XML_FEATURE_CONTEXT_BYTES, XML_L("XML_CONTEXT_BYTES"), - XML_CONTEXT_BYTES}, + {XML_FEATURE_CONTEXT_BYTES, XML_L("XML_CONTEXT_BYTES"), + XML_CONTEXT_BYTES}, #endif #ifdef XML_MIN_SIZE - {XML_FEATURE_MIN_SIZE, XML_L("XML_MIN_SIZE"), 0}, + {XML_FEATURE_MIN_SIZE, XML_L("XML_MIN_SIZE"), 0}, #endif #ifdef XML_NS - {XML_FEATURE_NS, XML_L("XML_NS"), 0}, + {XML_FEATURE_NS, XML_L("XML_NS"), 0}, #endif #ifdef XML_LARGE_SIZE - {XML_FEATURE_LARGE_SIZE, XML_L("XML_LARGE_SIZE"), 0}, + {XML_FEATURE_LARGE_SIZE, XML_L("XML_LARGE_SIZE"), 0}, #endif #ifdef XML_ATTR_INFO - {XML_FEATURE_ATTR_INFO, XML_L("XML_ATTR_INFO"), 0}, + {XML_FEATURE_ATTR_INFO, XML_L("XML_ATTR_INFO"), 0}, +#endif +#ifdef XML_DTD + /* Added in Expat 2.4.0. */ + {XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT, + XML_L("XML_BLAP_MAX_AMP"), + (long int) + EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT}, + {XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT, + XML_L("XML_BLAP_ACT_THRES"), + EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT}, #endif - {XML_FEATURE_END, NULL, 0}}; + {XML_FEATURE_END, NULL, 0}}; return features; } +#ifdef XML_DTD +XML_Bool XMLCALL +XML_SetBillionLaughsAttackProtectionMaximumAmplification( + XML_Parser parser, float maximumAmplificationFactor) { + if ((parser == NULL) || (parser->m_parentParser != NULL) + || isnan(maximumAmplificationFactor) + || (maximumAmplificationFactor < 1.0f)) { + return XML_FALSE; + } + parser->m_accounting.maximumAmplificationFactor = maximumAmplificationFactor; + return XML_TRUE; +} + +XML_Bool XMLCALL +XML_SetBillionLaughsAttackProtectionActivationThreshold( + XML_Parser parser, unsigned long long activationThresholdBytes) { + if ((parser == NULL) || (parser->m_parentParser != NULL)) { + return XML_FALSE; + } + parser->m_accounting.activationThresholdBytes = activationThresholdBytes; + return XML_TRUE; +} +#endif /* XML_DTD */ + /* Initially tag->rawName always points into the parse buffer; for those TAG instances opened while the current parse buffer was processed, and not yet closed, we need to store tag->rawName in a more @@ -2439,9 +2597,9 @@ storeRawNames(XML_Parser parser) { static enum XML_Error PTRCALL contentProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { - enum XML_Error result - = doContent(parser, 0, parser->m_encoding, start, end, endPtr, - (XML_Bool)! parser->m_parsingStatus.finalBuffer); + enum XML_Error result = doContent( + parser, 0, parser->m_encoding, start, end, endPtr, + (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_ACCOUNT_DIRECT); if (result == XML_ERROR_NONE) { if (! storeRawNames(parser)) return XML_ERROR_NO_MEMORY; @@ -2466,6 +2624,14 @@ externalEntityInitProcessor2(XML_Parser parser, const char *start, int tok = XmlContentTok(parser->m_encoding, start, end, &next); switch (tok) { case XML_TOK_BOM: +#ifdef XML_DTD + if (! accountingDiffTolerated(parser, tok, start, next, __LINE__, + XML_ACCOUNT_DIRECT)) { + accountingOnAbort(parser); + return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + } +#endif /* XML_DTD */ + /* If we are at the end of the buffer, this would cause the next stage, i.e. externalEntityInitProcessor3, to pass control directly to doContent (by detecting XML_TOK_NONE) without processing any xml text @@ -2503,6 +2669,10 @@ externalEntityInitProcessor3(XML_Parser parser, const char *start, const char *next = start; /* XmlContentTok doesn't always set the last arg */ parser->m_eventPtr = start; tok = XmlContentTok(parser->m_encoding, start, end, &next); + /* Note: These bytes are accounted later in: + - processXmlDecl + - externalEntityContentProcessor + */ parser->m_eventEndPtr = next; switch (tok) { @@ -2544,7 +2714,8 @@ externalEntityContentProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { enum XML_Error result = doContent(parser, 1, parser->m_encoding, start, end, endPtr, - (XML_Bool)! parser->m_parsingStatus.finalBuffer); + (XML_Bool)! parser->m_parsingStatus.finalBuffer, + XML_ACCOUNT_ENTITY_EXPANSION); if (result == XML_ERROR_NONE) { if (! storeRawNames(parser)) return XML_ERROR_NO_MEMORY; @@ -2555,7 +2726,7 @@ externalEntityContentProcessor(XML_Parser parser, const char *start, static enum XML_Error doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, const char *s, const char *end, const char **nextPtr, - XML_Bool haveMore) { + XML_Bool haveMore, enum XML_Account account) { /* save one level of indirection */ DTD *const dtd = parser->m_dtd; @@ -2573,6 +2744,17 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, for (;;) { const char *next = s; /* XmlContentTok doesn't always set the last arg */ int tok = XmlContentTok(enc, s, end, &next); +#ifdef XML_DTD + const char *accountAfter + = ((tok == XML_TOK_TRAILING_RSQB) || (tok == XML_TOK_TRAILING_CR)) + ? (haveMore ? s /* i.e. 0 bytes */ : end) + : next; + if (! accountingDiffTolerated(parser, tok, s, accountAfter, __LINE__, + account)) { + accountingOnAbort(parser); + return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + } +#endif *eventEndPP = next; switch (tok) { case XML_TOK_TRAILING_CR: @@ -2628,6 +2810,14 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, XML_Char ch = (XML_Char)XmlPredefinedEntityName( enc, s + enc->minBytesPerChar, next - enc->minBytesPerChar); if (ch) { +#ifdef XML_DTD + /* NOTE: We are replacing 4-6 characters original input for 1 character + * so there is no amplification and hence recording without + * protection. */ + accountingDiffTolerated(parser, tok, (char *)&ch, + ((char *)&ch) + sizeof(XML_Char), __LINE__, + XML_ACCOUNT_ENTITY_EXPANSION); +#endif /* XML_DTD */ if (parser->m_characterDataHandler) parser->m_characterDataHandler(parser->m_handlerArg, &ch, 1); else if (parser->m_defaultHandler) @@ -2746,7 +2936,8 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, } tag->name.str = (XML_Char *)tag->buf; *toPtr = XML_T('\0'); - result = storeAtts(parser, enc, s, &(tag->name), &(tag->bindings)); + result + = storeAtts(parser, enc, s, &(tag->name), &(tag->bindings), account); if (result) return result; if (parser->m_startElementHandler) @@ -2770,7 +2961,8 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, if (! name.str) return XML_ERROR_NO_MEMORY; poolFinish(&parser->m_tempPool); - result = storeAtts(parser, enc, s, &name, &bindings); + result = storeAtts(parser, enc, s, &name, &bindings, + XML_ACCOUNT_NONE /* token spans whole start tag */); if (result != XML_ERROR_NONE) { freeBindings(parser, bindings); return result; @@ -2905,7 +3097,8 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, /* END disabled code */ else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); - result = doCdataSection(parser, enc, &next, end, nextPtr, haveMore); + result + = doCdataSection(parser, enc, &next, end, nextPtr, haveMore, account); if (result != XML_ERROR_NONE) return result; else if (! next) { @@ -3034,7 +3227,8 @@ freeBindings(XML_Parser parser, BINDING *bindings) { */ static enum XML_Error storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, - TAG_NAME *tagNamePtr, BINDING **bindingsPtr) { + TAG_NAME *tagNamePtr, BINDING **bindingsPtr, + enum XML_Account account) { DTD *const dtd = parser->m_dtd; /* save one level of indirection */ ELEMENT_TYPE *elementType; int nDefaultAtts; @@ -3144,7 +3338,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, /* normalize the attribute value */ result = storeAttributeValue( parser, enc, isCdata, parser->m_atts[i].valuePtr, - parser->m_atts[i].valueEnd, &parser->m_tempPool); + parser->m_atts[i].valueEnd, &parser->m_tempPool, account); if (result) return result; appAtts[attIndex] = poolStart(&parser->m_tempPool); @@ -3533,9 +3727,9 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, static enum XML_Error PTRCALL cdataSectionProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { - enum XML_Error result - = doCdataSection(parser, parser->m_encoding, &start, end, endPtr, - (XML_Bool)! parser->m_parsingStatus.finalBuffer); + enum XML_Error result = doCdataSection( + parser, parser->m_encoding, &start, end, endPtr, + (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_ACCOUNT_DIRECT); if (result != XML_ERROR_NONE) return result; if (start) { @@ -3555,7 +3749,8 @@ cdataSectionProcessor(XML_Parser parser, const char *start, const char *end, */ static enum XML_Error doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, - const char *end, const char **nextPtr, XML_Bool haveMore) { + const char *end, const char **nextPtr, XML_Bool haveMore, + enum XML_Account account) { const char *s = *startPtr; const char **eventPP; const char **eventEndPP; @@ -3571,8 +3766,16 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, *startPtr = NULL; for (;;) { - const char *next; + const char *next = s; /* in case of XML_TOK_NONE or XML_TOK_PARTIAL */ int tok = XmlCdataSectionTok(enc, s, end, &next); +#ifdef XML_DTD + if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, account)) { + accountingOnAbort(parser); + return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + } +#else + UNUSED_P(account); +#endif *eventEndPP = next; switch (tok) { case XML_TOK_CDATA_SECT_CLOSE: @@ -3689,7 +3892,7 @@ ignoreSectionProcessor(XML_Parser parser, const char *start, const char *end, static enum XML_Error doIgnoreSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, const char *end, const char **nextPtr, XML_Bool haveMore) { - const char *next; + const char *next = *startPtr; /* in case of XML_TOK_NONE or XML_TOK_PARTIAL */ int tok; const char *s = *startPtr; const char **eventPP; @@ -3717,6 +3920,13 @@ doIgnoreSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, *eventPP = s; *startPtr = NULL; tok = XmlIgnoreSectionTok(enc, s, end, &next); +# ifdef XML_DTD + if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, + XML_ACCOUNT_DIRECT)) { + accountingOnAbort(parser); + return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + } +# endif *eventEndPP = next; switch (tok) { case XML_TOK_IGNORE_SECT: @@ -3801,6 +4011,15 @@ processXmlDecl(XML_Parser parser, int isGeneralTextEntity, const char *s, const char *versionend; const XML_Char *storedversion = NULL; int standalone = -1; + +#ifdef XML_DTD + if (! accountingDiffTolerated(parser, XML_TOK_XML_DECL, s, next, __LINE__, + XML_ACCOUNT_DIRECT)) { + accountingOnAbort(parser); + return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + } +#endif + if (! (parser->m_ns ? XmlParseXmlDeclNS : XmlParseXmlDecl)( isGeneralTextEntity, parser->m_encoding, s, next, &parser->m_eventPtr, &version, &versionend, &encodingName, &newEncoding, &standalone)) { @@ -3950,6 +4169,10 @@ entityValueInitProcessor(XML_Parser parser, const char *s, const char *end, for (;;) { tok = XmlPrologTok(parser->m_encoding, start, end, &next); + /* Note: Except for XML_TOK_BOM below, these bytes are accounted later in: + - storeEntityValue + - processXmlDecl + */ parser->m_eventEndPtr = next; if (tok <= 0) { if (! parser->m_parsingStatus.finalBuffer && tok != XML_TOK_INVALID) { @@ -3968,7 +4191,8 @@ entityValueInitProcessor(XML_Parser parser, const char *s, const char *end, break; } /* found end of entity value - can store it now */ - return storeEntityValue(parser, parser->m_encoding, s, end); + return storeEntityValue(parser, parser->m_encoding, s, end, + XML_ACCOUNT_DIRECT); } else if (tok == XML_TOK_XML_DECL) { enum XML_Error result; result = processXmlDecl(parser, 0, start, next); @@ -3995,6 +4219,14 @@ entityValueInitProcessor(XML_Parser parser, const char *s, const char *end, */ else if (tok == XML_TOK_BOM && next == end && ! parser->m_parsingStatus.finalBuffer) { +# ifdef XML_DTD + if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, + XML_ACCOUNT_DIRECT)) { + accountingOnAbort(parser); + return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + } +# endif + *nextPtr = next; return XML_ERROR_NONE; } @@ -4037,16 +4269,24 @@ externalParEntProcessor(XML_Parser parser, const char *s, const char *end, } /* This would cause the next stage, i.e. doProlog to be passed XML_TOK_BOM. However, when parsing an external subset, doProlog will not accept a BOM - as valid, and report a syntax error, so we have to skip the BOM + as valid, and report a syntax error, so we have to skip the BOM, and + account for the BOM bytes. */ else if (tok == XML_TOK_BOM) { + if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, + XML_ACCOUNT_DIRECT)) { + accountingOnAbort(parser); + return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + } + s = next; tok = XmlPrologTok(parser->m_encoding, s, end, &next); } parser->m_processor = prologProcessor; return doProlog(parser, parser->m_encoding, s, end, tok, next, nextPtr, - (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE); + (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE, + XML_ACCOUNT_DIRECT); } static enum XML_Error PTRCALL @@ -4059,6 +4299,9 @@ entityValueProcessor(XML_Parser parser, const char *s, const char *end, for (;;) { tok = XmlPrologTok(enc, start, end, &next); + /* Note: These bytes are accounted later in: + - storeEntityValue + */ if (tok <= 0) { if (! parser->m_parsingStatus.finalBuffer && tok != XML_TOK_INVALID) { *nextPtr = s; @@ -4076,7 +4319,7 @@ entityValueProcessor(XML_Parser parser, const char *s, const char *end, break; } /* found end of entity value - can store it now */ - return storeEntityValue(parser, enc, s, end); + return storeEntityValue(parser, enc, s, end, XML_ACCOUNT_DIRECT); } start = next; } @@ -4090,13 +4333,14 @@ prologProcessor(XML_Parser parser, const char *s, const char *end, const char *next = s; int tok = XmlPrologTok(parser->m_encoding, s, end, &next); return doProlog(parser, parser->m_encoding, s, end, tok, next, nextPtr, - (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE); + (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE, + XML_ACCOUNT_DIRECT); } static enum XML_Error doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, int tok, const char *next, const char **nextPtr, XML_Bool haveMore, - XML_Bool allowClosingDoctype) { + XML_Bool allowClosingDoctype, enum XML_Account account) { #ifdef XML_DTD static const XML_Char externalSubsetName[] = {ASCII_HASH, '\0'}; #endif /* XML_DTD */ @@ -4123,6 +4367,10 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, static const XML_Char enumValueSep[] = {ASCII_PIPE, '\0'}; static const XML_Char enumValueStart[] = {ASCII_LPAREN, '\0'}; +#ifndef XML_DTD + UNUSED_P(account); +#endif + /* save one level of indirection */ DTD *const dtd = parser->m_dtd; @@ -4187,6 +4435,19 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } } role = XmlTokenRole(&parser->m_prologState, tok, s, next, enc); +#ifdef XML_DTD + switch (role) { + case XML_ROLE_INSTANCE_START: // bytes accounted in contentProcessor + case XML_ROLE_XML_DECL: // bytes accounted in processXmlDecl + case XML_ROLE_TEXT_DECL: // bytes accounted in processXmlDecl + break; + default: + if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, account)) { + accountingOnAbort(parser); + return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + } + } +#endif switch (role) { case XML_ROLE_XML_DECL: { enum XML_Error result = processXmlDecl(parser, 0, s, next); @@ -4462,7 +4723,8 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, const XML_Char *attVal; enum XML_Error result = storeAttributeValue( parser, enc, parser->m_declAttributeIsCdata, - s + enc->minBytesPerChar, next - enc->minBytesPerChar, &dtd->pool); + s + enc->minBytesPerChar, next - enc->minBytesPerChar, &dtd->pool, + XML_ACCOUNT_NONE); if (result) return result; attVal = poolStart(&dtd->pool); @@ -4495,8 +4757,9 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, break; case XML_ROLE_ENTITY_VALUE: if (dtd->keepProcessing) { - enum XML_Error result = storeEntityValue( - parser, enc, s + enc->minBytesPerChar, next - enc->minBytesPerChar); + enum XML_Error result + = storeEntityValue(parser, enc, s + enc->minBytesPerChar, + next - enc->minBytesPerChar, XML_ACCOUNT_NONE); if (parser->m_declEntity) { parser->m_declEntity->textPtr = poolStart(&dtd->entityValuePool); parser->m_declEntity->textLen @@ -4886,12 +5149,15 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (parser->m_externalEntityRefHandler) { dtd->paramEntityRead = XML_FALSE; entity->open = XML_TRUE; + entityTrackingOnOpen(parser, entity, __LINE__); if (! parser->m_externalEntityRefHandler( parser->m_externalEntityRefHandlerArg, 0, entity->base, entity->systemId, entity->publicId)) { + entityTrackingOnClose(parser, entity, __LINE__); entity->open = XML_FALSE; return XML_ERROR_EXTERNAL_ENTITY_HANDLING; } + entityTrackingOnClose(parser, entity, __LINE__); entity->open = XML_FALSE; handleDefault = XML_FALSE; if (! dtd->paramEntityRead) { @@ -5089,6 +5355,13 @@ epilogProcessor(XML_Parser parser, const char *s, const char *end, for (;;) { const char *next = NULL; int tok = XmlPrologTok(parser->m_encoding, s, end, &next); +#ifdef XML_DTD + if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, + XML_ACCOUNT_DIRECT)) { + accountingOnAbort(parser); + return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + } +#endif parser->m_eventEndPtr = next; switch (tok) { /* report partial linebreak - it might be the last token */ @@ -5162,6 +5435,9 @@ processInternalEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl) { return XML_ERROR_NO_MEMORY; } entity->open = XML_TRUE; +#ifdef XML_DTD + entityTrackingOnOpen(parser, entity, __LINE__); +#endif entity->processed = 0; openEntity->next = parser->m_openInternalEntities; parser->m_openInternalEntities = openEntity; @@ -5170,8 +5446,8 @@ processInternalEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl) { openEntity->betweenDecl = betweenDecl; openEntity->internalEventPtr = NULL; openEntity->internalEventEndPtr = NULL; - textStart = (char *)entity->textPtr; - textEnd = (char *)(entity->textPtr + entity->textLen); + textStart = (const char *)entity->textPtr; + textEnd = (const char *)(entity->textPtr + entity->textLen); /* Set a safe default value in case 'next' does not get set */ next = textStart; @@ -5180,17 +5456,22 @@ processInternalEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl) { int tok = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next); result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd, - tok, next, &next, XML_FALSE, XML_FALSE); + tok, next, &next, XML_FALSE, XML_FALSE, + XML_ACCOUNT_ENTITY_EXPANSION); } else #endif /* XML_DTD */ result = doContent(parser, parser->m_tagLevel, parser->m_internalEncoding, - textStart, textEnd, &next, XML_FALSE); + textStart, textEnd, &next, XML_FALSE, + XML_ACCOUNT_ENTITY_EXPANSION); if (result == XML_ERROR_NONE) { if (textEnd != next && parser->m_parsingStatus.parsing == XML_SUSPENDED) { entity->processed = (int)(next - textStart); parser->m_processor = internalEntityProcessor; } else { +#ifdef XML_DTD + entityTrackingOnClose(parser, entity, __LINE__); +#endif /* XML_DTD */ entity->open = XML_FALSE; parser->m_openInternalEntities = openEntity->next; /* put openEntity back in list of free instances */ @@ -5213,8 +5494,8 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, return XML_ERROR_UNEXPECTED_STATE; entity = openEntity->entity; - textStart = ((char *)entity->textPtr) + entity->processed; - textEnd = (char *)(entity->textPtr + entity->textLen); + textStart = ((const char *)entity->textPtr) + entity->processed; + textEnd = (const char *)(entity->textPtr + entity->textLen); /* Set a safe default value in case 'next' does not get set */ next = textStart; @@ -5223,20 +5504,24 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, int tok = XmlPrologTok(parser->m_internalEncoding, textStart, textEnd, &next); result = doProlog(parser, parser->m_internalEncoding, textStart, textEnd, - tok, next, &next, XML_FALSE, XML_TRUE); + tok, next, &next, XML_FALSE, XML_TRUE, + XML_ACCOUNT_ENTITY_EXPANSION); } else #endif /* XML_DTD */ result = doContent(parser, openEntity->startTagLevel, parser->m_internalEncoding, textStart, textEnd, &next, - XML_FALSE); + XML_FALSE, XML_ACCOUNT_ENTITY_EXPANSION); if (result != XML_ERROR_NONE) return result; else if (textEnd != next && parser->m_parsingStatus.parsing == XML_SUSPENDED) { - entity->processed = (int)(next - (char *)entity->textPtr); + entity->processed = (int)(next - (const char *)entity->textPtr); return result; } else { +#ifdef XML_DTD + entityTrackingOnClose(parser, entity, __LINE__); +#endif entity->open = XML_FALSE; parser->m_openInternalEntities = openEntity->next; /* put openEntity back in list of free instances */ @@ -5250,7 +5535,8 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, parser->m_processor = prologProcessor; tok = XmlPrologTok(parser->m_encoding, s, end, &next); return doProlog(parser, parser->m_encoding, s, end, tok, next, nextPtr, - (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE); + (XML_Bool)! parser->m_parsingStatus.finalBuffer, XML_TRUE, + XML_ACCOUNT_DIRECT); } else #endif /* XML_DTD */ { @@ -5258,7 +5544,8 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, /* see externalEntityContentProcessor vs contentProcessor */ return doContent(parser, parser->m_parentParser ? 1 : 0, parser->m_encoding, s, end, nextPtr, - (XML_Bool)! parser->m_parsingStatus.finalBuffer); + (XML_Bool)! parser->m_parsingStatus.finalBuffer, + XML_ACCOUNT_DIRECT); } } @@ -5273,9 +5560,10 @@ errorProcessor(XML_Parser parser, const char *s, const char *end, static enum XML_Error storeAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, - const char *ptr, const char *end, STRING_POOL *pool) { + const char *ptr, const char *end, STRING_POOL *pool, + enum XML_Account account) { enum XML_Error result - = appendAttributeValue(parser, enc, isCdata, ptr, end, pool); + = appendAttributeValue(parser, enc, isCdata, ptr, end, pool, account); if (result) return result; if (! isCdata && poolLength(pool) && poolLastChar(pool) == 0x20) @@ -5287,11 +5575,23 @@ storeAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, static enum XML_Error appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, - const char *ptr, const char *end, STRING_POOL *pool) { + const char *ptr, const char *end, STRING_POOL *pool, + enum XML_Account account) { DTD *const dtd = parser->m_dtd; /* save one level of indirection */ +#ifndef XML_DTD + UNUSED_P(account); +#endif + for (;;) { - const char *next; + const char *next + = ptr; /* XmlAttributeValueTok doesn't always set the last arg */ int tok = XmlAttributeValueTok(enc, ptr, end, &next); +#ifdef XML_DTD + if (! accountingDiffTolerated(parser, tok, ptr, next, __LINE__, account)) { + accountingOnAbort(parser); + return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + } +#endif switch (tok) { case XML_TOK_NONE: return XML_ERROR_NONE; @@ -5351,6 +5651,14 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, XML_Char ch = (XML_Char)XmlPredefinedEntityName( enc, ptr + enc->minBytesPerChar, next - enc->minBytesPerChar); if (ch) { +#ifdef XML_DTD + /* NOTE: We are replacing 4-6 characters original input for 1 character + * so there is no amplification and hence recording without + * protection. */ + accountingDiffTolerated(parser, tok, (char *)&ch, + ((char *)&ch) + sizeof(XML_Char), __LINE__, + XML_ACCOUNT_ENTITY_EXPANSION); +#endif /* XML_DTD */ if (! poolAppendChar(pool, ch)) return XML_ERROR_NO_MEMORY; break; @@ -5428,9 +5736,16 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, enum XML_Error result; const XML_Char *textEnd = entity->textPtr + entity->textLen; entity->open = XML_TRUE; +#ifdef XML_DTD + entityTrackingOnOpen(parser, entity, __LINE__); +#endif result = appendAttributeValue(parser, parser->m_internalEncoding, - isCdata, (char *)entity->textPtr, - (char *)textEnd, pool); + isCdata, (const char *)entity->textPtr, + (const char *)textEnd, pool, + XML_ACCOUNT_ENTITY_EXPANSION); +#ifdef XML_DTD + entityTrackingOnClose(parser, entity, __LINE__); +#endif entity->open = XML_FALSE; if (result) return result; @@ -5460,13 +5775,16 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, static enum XML_Error storeEntityValue(XML_Parser parser, const ENCODING *enc, - const char *entityTextPtr, const char *entityTextEnd) { + const char *entityTextPtr, const char *entityTextEnd, + enum XML_Account account) { DTD *const dtd = parser->m_dtd; /* save one level of indirection */ STRING_POOL *pool = &(dtd->entityValuePool); enum XML_Error result = XML_ERROR_NONE; #ifdef XML_DTD int oldInEntityValue = parser->m_prologState.inEntityValue; parser->m_prologState.inEntityValue = 1; +#else + UNUSED_P(account); #endif /* XML_DTD */ /* never return Null for the value argument in EntityDeclHandler, since this would indicate an external entity; therefore we @@ -5477,8 +5795,19 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, } for (;;) { - const char *next; + const char *next + = entityTextPtr; /* XmlEntityValueTok doesn't always set the last arg */ int tok = XmlEntityValueTok(enc, entityTextPtr, entityTextEnd, &next); + +#ifdef XML_DTD + if (! accountingDiffTolerated(parser, tok, entityTextPtr, next, __LINE__, + account)) { + accountingOnAbort(parser); + result = XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + goto endEntityValue; + } +#endif + switch (tok) { case XML_TOK_PARAM_ENTITY_REF: #ifdef XML_DTD @@ -5514,13 +5843,16 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, if (parser->m_externalEntityRefHandler) { dtd->paramEntityRead = XML_FALSE; entity->open = XML_TRUE; + entityTrackingOnOpen(parser, entity, __LINE__); if (! parser->m_externalEntityRefHandler( parser->m_externalEntityRefHandlerArg, 0, entity->base, entity->systemId, entity->publicId)) { + entityTrackingOnClose(parser, entity, __LINE__); entity->open = XML_FALSE; result = XML_ERROR_EXTERNAL_ENTITY_HANDLING; goto endEntityValue; } + entityTrackingOnClose(parser, entity, __LINE__); entity->open = XML_FALSE; if (! dtd->paramEntityRead) dtd->keepProcessing = dtd->standalone; @@ -5528,9 +5860,12 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, dtd->keepProcessing = dtd->standalone; } else { entity->open = XML_TRUE; + entityTrackingOnOpen(parser, entity, __LINE__); result = storeEntityValue( - parser, parser->m_internalEncoding, (char *)entity->textPtr, - (char *)(entity->textPtr + entity->textLen)); + parser, parser->m_internalEncoding, (const char *)entity->textPtr, + (const char *)(entity->textPtr + entity->textLen), + XML_ACCOUNT_ENTITY_EXPANSION); + entityTrackingOnClose(parser, entity, __LINE__); entity->open = XML_FALSE; if (result) goto endEntityValue; @@ -6485,7 +6820,7 @@ hashTableInit(HASH_TABLE *p, const XML_Memory_Handling_Suite *ms) { static void FASTCALL hashTableIterInit(HASH_TABLE_ITER *iter, const HASH_TABLE *table) { iter->p = table->v; - iter->end = iter->p + table->size; + iter->end = iter->p ? iter->p + table->size : NULL; } static NAMED *FASTCALL @@ -6891,3 +7226,755 @@ copyString(const XML_Char *s, const XML_Memory_Handling_Suite *memsuite) { memcpy(result, s, charsRequired * sizeof(XML_Char)); return result; } + +#ifdef XML_DTD + +static float +accountingGetCurrentAmplification(XML_Parser rootParser) { + const XmlBigCount countBytesOutput + = rootParser->m_accounting.countBytesDirect + + rootParser->m_accounting.countBytesIndirect; + const float amplificationFactor + = rootParser->m_accounting.countBytesDirect + ? (countBytesOutput + / (float)(rootParser->m_accounting.countBytesDirect)) + : 1.0f; + assert(! rootParser->m_parentParser); + return amplificationFactor; +} + +static void +accountingReportStats(XML_Parser originParser, const char *epilog) { + const XML_Parser rootParser = getRootParserOf(originParser, NULL); + assert(! rootParser->m_parentParser); + + if (rootParser->m_accounting.debugLevel < 1) { + return; + } + + const float amplificationFactor + = accountingGetCurrentAmplification(rootParser); + fprintf(stderr, + "expat: Accounting(%p): Direct " EXPAT_FMT_ULL( + "10") ", indirect " EXPAT_FMT_ULL("10") ", amplification %8.2f%s", + (void *)rootParser, rootParser->m_accounting.countBytesDirect, + rootParser->m_accounting.countBytesIndirect, + (double)amplificationFactor, epilog); +} + +static void +accountingOnAbort(XML_Parser originParser) { + accountingReportStats(originParser, " ABORTING\n"); +} + +static void +accountingReportDiff(XML_Parser rootParser, + unsigned int levelsAwayFromRootParser, const char *before, + const char *after, ptrdiff_t bytesMore, int source_line, + enum XML_Account account) { + assert(! rootParser->m_parentParser); + + fprintf(stderr, + " (+" EXPAT_FMT_PTRDIFF_T("6") " bytes %s|%d, xmlparse.c:%d) %*s\"", + bytesMore, (account == XML_ACCOUNT_DIRECT) ? "DIR" : "EXP", + levelsAwayFromRootParser, source_line, 10, ""); + + const char ellipis[] = "[..]"; + const size_t ellipsisLength = sizeof(ellipis) /* because compile-time */ - 1; + const unsigned int contextLength = 10; + + /* Note: Performance is of no concern here */ + const char *walker = before; + if ((rootParser->m_accounting.debugLevel >= 3) + || (after - before) + <= (ptrdiff_t)(contextLength + ellipsisLength + contextLength)) { + for (; walker < after; walker++) { + fprintf(stderr, "%s", unsignedCharToPrintable(walker[0])); + } + } else { + for (; walker < before + contextLength; walker++) { + fprintf(stderr, "%s", unsignedCharToPrintable(walker[0])); + } + fprintf(stderr, ellipis); + walker = after - contextLength; + for (; walker < after; walker++) { + fprintf(stderr, "%s", unsignedCharToPrintable(walker[0])); + } + } + fprintf(stderr, "\"\n"); +} + +static XML_Bool +accountingDiffTolerated(XML_Parser originParser, int tok, const char *before, + const char *after, int source_line, + enum XML_Account account) { + /* Note: We need to check the token type *first* to be sure that + * we can even access variable <after>, safely. + * E.g. for XML_TOK_NONE <after> may hold an invalid pointer. */ + switch (tok) { + case XML_TOK_INVALID: + case XML_TOK_PARTIAL: + case XML_TOK_PARTIAL_CHAR: + case XML_TOK_NONE: + return XML_TRUE; + } + + if (account == XML_ACCOUNT_NONE) + return XML_TRUE; /* because these bytes have been accounted for, already */ + + unsigned int levelsAwayFromRootParser; + const XML_Parser rootParser + = getRootParserOf(originParser, &levelsAwayFromRootParser); + assert(! rootParser->m_parentParser); + + const int isDirect + = (account == XML_ACCOUNT_DIRECT) && (originParser == rootParser); + const ptrdiff_t bytesMore = after - before; + + XmlBigCount *const additionTarget + = isDirect ? &rootParser->m_accounting.countBytesDirect + : &rootParser->m_accounting.countBytesIndirect; + + /* Detect and avoid integer overflow */ + if (*additionTarget > (XmlBigCount)(-1) - (XmlBigCount)bytesMore) + return XML_FALSE; + *additionTarget += bytesMore; + + const XmlBigCount countBytesOutput + = rootParser->m_accounting.countBytesDirect + + rootParser->m_accounting.countBytesIndirect; + const float amplificationFactor + = accountingGetCurrentAmplification(rootParser); + const XML_Bool tolerated + = (countBytesOutput < rootParser->m_accounting.activationThresholdBytes) + || (amplificationFactor + <= rootParser->m_accounting.maximumAmplificationFactor); + + if (rootParser->m_accounting.debugLevel >= 2) { + accountingReportStats(rootParser, ""); + accountingReportDiff(rootParser, levelsAwayFromRootParser, before, after, + bytesMore, source_line, account); + } + + return tolerated; +} + +unsigned long long +testingAccountingGetCountBytesDirect(XML_Parser parser) { + if (! parser) + return 0; + return parser->m_accounting.countBytesDirect; +} + +unsigned long long +testingAccountingGetCountBytesIndirect(XML_Parser parser) { + if (! parser) + return 0; + return parser->m_accounting.countBytesIndirect; +} + +static void +entityTrackingReportStats(XML_Parser rootParser, ENTITY *entity, + const char *action, int sourceLine) { + assert(! rootParser->m_parentParser); + if (rootParser->m_entity_stats.debugLevel < 1) + return; + +# if defined(XML_UNICODE) + const char *const entityName = "[..]"; +# else + const char *const entityName = entity->name; +# endif + + fprintf( + stderr, + "expat: Entities(%p): Count %9d, depth %2d/%2d %*s%s%s; %s length %d (xmlparse.c:%d)\n", + (void *)rootParser, rootParser->m_entity_stats.countEverOpened, + rootParser->m_entity_stats.currentDepth, + rootParser->m_entity_stats.maximumDepthSeen, + (rootParser->m_entity_stats.currentDepth - 1) * 2, "", + entity->is_param ? "%" : "&", entityName, action, entity->textLen, + sourceLine); +} + +static void +entityTrackingOnOpen(XML_Parser originParser, ENTITY *entity, int sourceLine) { + const XML_Parser rootParser = getRootParserOf(originParser, NULL); + assert(! rootParser->m_parentParser); + + rootParser->m_entity_stats.countEverOpened++; + rootParser->m_entity_stats.currentDepth++; + if (rootParser->m_entity_stats.currentDepth + > rootParser->m_entity_stats.maximumDepthSeen) { + rootParser->m_entity_stats.maximumDepthSeen++; + } + + entityTrackingReportStats(rootParser, entity, "OPEN ", sourceLine); +} + +static void +entityTrackingOnClose(XML_Parser originParser, ENTITY *entity, int sourceLine) { + const XML_Parser rootParser = getRootParserOf(originParser, NULL); + assert(! rootParser->m_parentParser); + + entityTrackingReportStats(rootParser, entity, "CLOSE", sourceLine); + rootParser->m_entity_stats.currentDepth--; +} + +static XML_Parser +getRootParserOf(XML_Parser parser, unsigned int *outLevelDiff) { + XML_Parser rootParser = parser; + unsigned int stepsTakenUpwards = 0; + while (rootParser->m_parentParser) { + rootParser = rootParser->m_parentParser; + stepsTakenUpwards++; + } + assert(! rootParser->m_parentParser); + if (outLevelDiff != NULL) { + *outLevelDiff = stepsTakenUpwards; + } + return rootParser; +} + +const char * +unsignedCharToPrintable(unsigned char c) { + switch (c) { + case 0: + return "\\0"; + case 1: + return "\\x1"; + case 2: + return "\\x2"; + case 3: + return "\\x3"; + case 4: + return "\\x4"; + case 5: + return "\\x5"; + case 6: + return "\\x6"; + case 7: + return "\\x7"; + case 8: + return "\\x8"; + case 9: + return "\\t"; + case 10: + return "\\n"; + case 11: + return "\\xB"; + case 12: + return "\\xC"; + case 13: + return "\\r"; + case 14: + return "\\xE"; + case 15: + return "\\xF"; + case 16: + return "\\x10"; + case 17: + return "\\x11"; + case 18: + return "\\x12"; + case 19: + return "\\x13"; + case 20: + return "\\x14"; + case 21: + return "\\x15"; + case 22: + return "\\x16"; + case 23: + return "\\x17"; + case 24: + return "\\x18"; + case 25: + return "\\x19"; + case 26: + return "\\x1A"; + case 27: + return "\\x1B"; + case 28: + return "\\x1C"; + case 29: + return "\\x1D"; + case 30: + return "\\x1E"; + case 31: + return "\\x1F"; + case 32: + return " "; + case 33: + return "!"; + case 34: + return "\\\""; + case 35: + return "#"; + case 36: + return "$"; + case 37: + return "%"; + case 38: + return "&"; + case 39: + return "'"; + case 40: + return "("; + case 41: + return ")"; + case 42: + return "*"; + case 43: + return "+"; + case 44: + return ","; + case 45: + return "-"; + case 46: + return "."; + case 47: + return "/"; + case 48: + return "0"; + case 49: + return "1"; + case 50: + return "2"; + case 51: + return "3"; + case 52: + return "4"; + case 53: + return "5"; + case 54: + return "6"; + case 55: + return "7"; + case 56: + return "8"; + case 57: + return "9"; + case 58: + return ":"; + case 59: + return ";"; + case 60: + return "<"; + case 61: + return "="; + case 62: + return ">"; + case 63: + return "?"; + case 64: + return "@"; + case 65: + return "A"; + case 66: + return "B"; + case 67: + return "C"; + case 68: + return "D"; + case 69: + return "E"; + case 70: + return "F"; + case 71: + return "G"; + case 72: + return "H"; + case 73: + return "I"; + case 74: + return "J"; + case 75: + return "K"; + case 76: + return "L"; + case 77: + return "M"; + case 78: + return "N"; + case 79: + return "O"; + case 80: + return "P"; + case 81: + return "Q"; + case 82: + return "R"; + case 83: + return "S"; + case 84: + return "T"; + case 85: + return "U"; + case 86: + return "V"; + case 87: + return "W"; + case 88: + return "X"; + case 89: + return "Y"; + case 90: + return "Z"; + case 91: + return "["; + case 92: + return "\\\\"; + case 93: + return "]"; + case 94: + return "^"; + case 95: + return "_"; + case 96: + return "`"; + case 97: + return "a"; + case 98: + return "b"; + case 99: + return "c"; + case 100: + return "d"; + case 101: + return "e"; + case 102: + return "f"; + case 103: + return "g"; + case 104: + return "h"; + case 105: + return "i"; + case 106: + return "j"; + case 107: + return "k"; + case 108: + return "l"; + case 109: + return "m"; + case 110: + return "n"; + case 111: + return "o"; + case 112: + return "p"; + case 113: + return "q"; + case 114: + return "r"; + case 115: + return "s"; + case 116: + return "t"; + case 117: + return "u"; + case 118: + return "v"; + case 119: + return "w"; + case 120: + return "x"; + case 121: + return "y"; + case 122: + return "z"; + case 123: + return "{"; + case 124: + return "|"; + case 125: + return "}"; + case 126: + return "~"; + case 127: + return "\\x7F"; + case 128: + return "\\x80"; + case 129: + return "\\x81"; + case 130: + return "\\x82"; + case 131: + return "\\x83"; + case 132: + return "\\x84"; + case 133: + return "\\x85"; + case 134: + return "\\x86"; + case 135: + return "\\x87"; + case 136: + return "\\x88"; + case 137: + return "\\x89"; + case 138: + return "\\x8A"; + case 139: + return "\\x8B"; + case 140: + return "\\x8C"; + case 141: + return "\\x8D"; + case 142: + return "\\x8E"; + case 143: + return "\\x8F"; + case 144: + return "\\x90"; + case 145: + return "\\x91"; + case 146: + return "\\x92"; + case 147: + return "\\x93"; + case 148: + return "\\x94"; + case 149: + return "\\x95"; + case 150: + return "\\x96"; + case 151: + return "\\x97"; + case 152: + return "\\x98"; + case 153: + return "\\x99"; + case 154: + return "\\x9A"; + case 155: + return "\\x9B"; + case 156: + return "\\x9C"; + case 157: + return "\\x9D"; + case 158: + return "\\x9E"; + case 159: + return "\\x9F"; + case 160: + return "\\xA0"; + case 161: + return "\\xA1"; + case 162: + return "\\xA2"; + case 163: + return "\\xA3"; + case 164: + return "\\xA4"; + case 165: + return "\\xA5"; + case 166: + return "\\xA6"; + case 167: + return "\\xA7"; + case 168: + return "\\xA8"; + case 169: + return "\\xA9"; + case 170: + return "\\xAA"; + case 171: + return "\\xAB"; + case 172: + return "\\xAC"; + case 173: + return "\\xAD"; + case 174: + return "\\xAE"; + case 175: + return "\\xAF"; + case 176: + return "\\xB0"; + case 177: + return "\\xB1"; + case 178: + return "\\xB2"; + case 179: + return "\\xB3"; + case 180: + return "\\xB4"; + case 181: + return "\\xB5"; + case 182: + return "\\xB6"; + case 183: + return "\\xB7"; + case 184: + return "\\xB8"; + case 185: + return "\\xB9"; + case 186: + return "\\xBA"; + case 187: + return "\\xBB"; + case 188: + return "\\xBC"; + case 189: + return "\\xBD"; + case 190: + return "\\xBE"; + case 191: + return "\\xBF"; + case 192: + return "\\xC0"; + case 193: + return "\\xC1"; + case 194: + return "\\xC2"; + case 195: + return "\\xC3"; + case 196: + return "\\xC4"; + case 197: + return "\\xC5"; + case 198: + return "\\xC6"; + case 199: + return "\\xC7"; + case 200: + return "\\xC8"; + case 201: + return "\\xC9"; + case 202: + return "\\xCA"; + case 203: + return "\\xCB"; + case 204: + return "\\xCC"; + case 205: + return "\\xCD"; + case 206: + return "\\xCE"; + case 207: + return "\\xCF"; + case 208: + return "\\xD0"; + case 209: + return "\\xD1"; + case 210: + return "\\xD2"; + case 211: + return "\\xD3"; + case 212: + return "\\xD4"; + case 213: + return "\\xD5"; + case 214: + return "\\xD6"; + case 215: + return "\\xD7"; + case 216: + return "\\xD8"; + case 217: + return "\\xD9"; + case 218: + return "\\xDA"; + case 219: + return "\\xDB"; + case 220: + return "\\xDC"; + case 221: + return "\\xDD"; + case 222: + return "\\xDE"; + case 223: + return "\\xDF"; + case 224: + return "\\xE0"; + case 225: + return "\\xE1"; + case 226: + return "\\xE2"; + case 227: + return "\\xE3"; + case 228: + return "\\xE4"; + case 229: + return "\\xE5"; + case 230: + return "\\xE6"; + case 231: + return "\\xE7"; + case 232: + return "\\xE8"; + case 233: + return "\\xE9"; + case 234: + return "\\xEA"; + case 235: + return "\\xEB"; + case 236: + return "\\xEC"; + case 237: + return "\\xED"; + case 238: + return "\\xEE"; + case 239: + return "\\xEF"; + case 240: + return "\\xF0"; + case 241: + return "\\xF1"; + case 242: + return "\\xF2"; + case 243: + return "\\xF3"; + case 244: + return "\\xF4"; + case 245: + return "\\xF5"; + case 246: + return "\\xF6"; + case 247: + return "\\xF7"; + case 248: + return "\\xF8"; + case 249: + return "\\xF9"; + case 250: + return "\\xFA"; + case 251: + return "\\xFB"; + case 252: + return "\\xFC"; + case 253: + return "\\xFD"; + case 254: + return "\\xFE"; + case 255: + return "\\xFF"; + default: + assert(0); /* never gets here */ + return "dead code"; + } + assert(0); /* never gets here */ +} + +#endif /* XML_DTD */ + +static unsigned long +getDebugLevel(const char *variableName, unsigned long defaultDebugLevel) { + const char *const valueOrNull = getenv(variableName); + if (valueOrNull == NULL) { + return defaultDebugLevel; + } + const char *const value = valueOrNull; + + errno = 0; + char *afterValue = (char *)value; + unsigned long debugLevel = strtoul(value, &afterValue, 10); + if ((errno != 0) || (afterValue[0] != '\0')) { + errno = 0; + return defaultDebugLevel; + } + + return debugLevel; +} diff --git a/Modules/expat/xmlrole.c b/Modules/expat/xmlrole.c index 4d3e3e86e9e864..08173b0fd541d2 100644 --- a/Modules/expat/xmlrole.c +++ b/Modules/expat/xmlrole.c @@ -7,7 +7,14 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper <coopercc(a)users.sourceforge.net> + Copyright (c) 2002 Greg Stein <gstein(a)users.sourceforge.net> + Copyright (c) 2002-2006 Karl Waclawek <karl(a)waclawek.net> + Copyright (c) 2002-2003 Fred L. Drake, Jr. <fdrake(a)users.sourceforge.net> + Copyright (c) 2005-2009 Steven Solie <ssolie(a)users.sourceforge.net> + Copyright (c) 2016-2021 Sebastian Pipping <sebastian(a)pipping.org> + Copyright (c) 2017 Rhodri James <rhodri(a)wildebeest.org.uk> + Copyright (c) 2019 David Loffredo <loffredo(a)steptools.com> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -34,11 +41,9 @@ #ifdef _WIN32 # include "winconfig.h" -#else -# ifdef HAVE_EXPAT_CONFIG_H -# include <expat_config.h> -# endif -#endif /* ndef _WIN32 */ +#endif + +#include <expat_config.h> #include "expat_external.h" #include "internal.h" @@ -1220,6 +1225,8 @@ common(PROLOG_STATE *state, int tok) { #ifdef XML_DTD if (! state->documentEntity && tok == XML_TOK_PARAM_ENTITY_REF) return XML_ROLE_INNER_PARAM_ENTITY_REF; +#else + UNUSED_P(tok); #endif state->handler = error; return XML_ROLE_ERROR; diff --git a/Modules/expat/xmlrole.h b/Modules/expat/xmlrole.h index 036aba64fd29c6..d6e1fa150a108a 100644 --- a/Modules/expat/xmlrole.h +++ b/Modules/expat/xmlrole.h @@ -7,7 +7,10 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper <coopercc(a)users.sourceforge.net> + Copyright (c) 2002 Karl Waclawek <karl(a)waclawek.net> + Copyright (c) 2002 Fred L. Drake, Jr. <fdrake(a)users.sourceforge.net> + Copyright (c) 2017 Sebastian Pipping <sebastian(a)pipping.org> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining diff --git a/Modules/expat/xmltok.c b/Modules/expat/xmltok.c index 11e9d1ccdad423..f2b6b406067ea9 100644 --- a/Modules/expat/xmltok.c +++ b/Modules/expat/xmltok.c @@ -7,7 +7,19 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper <coopercc(a)users.sourceforge.net> + Copyright (c) 2001-2003 Fred L. Drake, Jr. <fdrake(a)users.sourceforge.net> + Copyright (c) 2002 Greg Stein <gstein(a)users.sourceforge.net> + Copyright (c) 2002-2016 Karl Waclawek <karl(a)waclawek.net> + Copyright (c) 2005-2009 Steven Solie <ssolie(a)users.sourceforge.net> + Copyright (c) 2016-2021 Sebastian Pipping <sebastian(a)pipping.org> + Copyright (c) 2016 Pascal Cuoq <cuoq(a)trust-in-soft.com> + Copyright (c) 2016 Don Lewis <truckman(a)apache.org> + Copyright (c) 2017 Rhodri James <rhodri(a)wildebeest.org.uk> + Copyright (c) 2017 Alexander Bluhm <alexander.bluhm(a)gmx.net> + Copyright (c) 2017 Benbuck Nason <bnason(a)netflix.com> + Copyright (c) 2017 José Gutiérrez de la Concha <jose(a)zeroc.com> + Copyright (c) 2019 David Loffredo <loffredo(a)steptools.com> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -32,23 +44,13 @@ #include <stddef.h> #include <string.h> /* memcpy */ - -#if defined(_MSC_VER) && (_MSC_VER <= 1700) -/* for vs2012/11.0/1700 and earlier Visual Studio compilers */ -# define bool int -# define false 0 -# define true 1 -#else -# include <stdbool.h> -#endif +#include <stdbool.h> #ifdef _WIN32 # include "winconfig.h" -#else -# ifdef HAVE_EXPAT_CONFIG_H -# include <expat_config.h> -# endif -#endif /* ndef _WIN32 */ +#endif + +#include <expat_config.h> #include "expat_external.h" #include "internal.h" @@ -269,8 +271,14 @@ sb_byteToAscii(const ENCODING *enc, const char *p) { #define IS_NAME_CHAR(enc, p, n) (AS_NORMAL_ENCODING(enc)->isName##n(enc, p)) #define IS_NMSTRT_CHAR(enc, p, n) (AS_NORMAL_ENCODING(enc)->isNmstrt##n(enc, p)) -#define IS_INVALID_CHAR(enc, p, n) \ - (AS_NORMAL_ENCODING(enc)->isInvalid##n(enc, p)) +#ifdef XML_MIN_SIZE +# define IS_INVALID_CHAR(enc, p, n) \ + (AS_NORMAL_ENCODING(enc)->isInvalid##n \ + && AS_NORMAL_ENCODING(enc)->isInvalid##n(enc, p)) +#else +# define IS_INVALID_CHAR(enc, p, n) \ + (AS_NORMAL_ENCODING(enc)->isInvalid##n(enc, p)) +#endif #ifdef XML_MIN_SIZE # define IS_NAME_CHAR_MINBPC(enc, p) \ @@ -589,13 +597,13 @@ static const struct normal_encoding ascii_encoding static int PTRFASTCALL unicode_byte_type(char hi, char lo) { switch ((unsigned char)hi) { - /* 0xD800–0xDBFF first 16-bit code unit or high surrogate (W1) */ + /* 0xD800-0xDBFF first 16-bit code unit or high surrogate (W1) */ case 0xD8: case 0xD9: case 0xDA: case 0xDB: return BT_LEAD4; - /* 0xDC00–0xDFFF second 16-bit code unit or low surrogate (W2) */ + /* 0xDC00-0xDFFF second 16-bit code unit or low surrogate (W2) */ case 0xDC: case 0xDD: case 0xDE: diff --git a/Modules/expat/xmltok.h b/Modules/expat/xmltok.h index 2adbf5307befae..6f630c2f9ba96d 100644 --- a/Modules/expat/xmltok.h +++ b/Modules/expat/xmltok.h @@ -7,7 +7,11 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper <coopercc(a)users.sourceforge.net> + Copyright (c) 2002 Fred L. Drake, Jr. <fdrake(a)users.sourceforge.net> + Copyright (c) 2002-2005 Karl Waclawek <karl(a)waclawek.net> + Copyright (c) 2016-2017 Sebastian Pipping <sebastian(a)pipping.org> + Copyright (c) 2017 Rhodri James <rhodri(a)wildebeest.org.uk> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining diff --git a/Modules/expat/xmltok_impl.c b/Modules/expat/xmltok_impl.c index c209221cd79d13..0430591b42636e 100644 --- a/Modules/expat/xmltok_impl.c +++ b/Modules/expat/xmltok_impl.c @@ -1,4 +1,4 @@ -/* This file is included! +/* This file is included (from xmltok.c, 1-3 times depending on XML_MIN_SIZE)! __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -7,7 +7,15 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper <coopercc(a)users.sourceforge.net> + Copyright (c) 2002 Fred L. Drake, Jr. <fdrake(a)users.sourceforge.net> + Copyright (c) 2002-2016 Karl Waclawek <karl(a)waclawek.net> + Copyright (c) 2016-2021 Sebastian Pipping <sebastian(a)pipping.org> + Copyright (c) 2017 Rhodri James <rhodri(a)wildebeest.org.uk> + Copyright (c) 2018 Benjamin Peterson <benjamin(a)python.org> + Copyright (c) 2018 Anton Maklakov <antmak.pub(a)gmail.com> + Copyright (c) 2019 David Loffredo <loffredo(a)steptools.com> + Copyright (c) 2020 Boris Kolpackov <boris(a)codesynthesis.com> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -32,7 +40,7 @@ #ifdef XML_TOK_IMPL_C -# ifndef IS_INVALID_CHAR +# ifndef IS_INVALID_CHAR // i.e. for UTF-16 and XML_MIN_SIZE not defined # define IS_INVALID_CHAR(enc, ptr, n) (0) # endif @@ -1768,13 +1776,14 @@ PREFIX(updatePosition)(const ENCODING *enc, const char *ptr, const char *end, # define LEAD_CASE(n) \ case BT_LEAD##n: \ ptr += n; \ + pos->columnNumber++; \ break; LEAD_CASE(2) LEAD_CASE(3) LEAD_CASE(4) # undef LEAD_CASE case BT_LF: - pos->columnNumber = (XML_Size)-1; + pos->columnNumber = 0; pos->lineNumber++; ptr += MINBPC(enc); break; @@ -1783,13 +1792,13 @@ PREFIX(updatePosition)(const ENCODING *enc, const char *ptr, const char *end, ptr += MINBPC(enc); if (HAS_CHAR(enc, ptr, end) && BYTE_TYPE(enc, ptr) == BT_LF) ptr += MINBPC(enc); - pos->columnNumber = (XML_Size)-1; + pos->columnNumber = 0; break; default: ptr += MINBPC(enc); + pos->columnNumber++; break; } - pos->columnNumber++; } } diff --git a/Modules/expat/xmltok_impl.h b/Modules/expat/xmltok_impl.h index e925dbc7e2c833..c518aada013df2 100644 --- a/Modules/expat/xmltok_impl.h +++ b/Modules/expat/xmltok_impl.h @@ -7,7 +7,8 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper <coopercc(a)users.sourceforge.net> + Copyright (c) 2017-2019 Sebastian Pipping <sebastian(a)pipping.org> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining diff --git a/Modules/expat/xmltok_ns.c b/Modules/expat/xmltok_ns.c index 919c74e9f97fe8..5fd83922359400 100644 --- a/Modules/expat/xmltok_ns.c +++ b/Modules/expat/xmltok_ns.c @@ -7,7 +7,11 @@ |_| XML parser Copyright (c) 1997-2000 Thai Open Source Software Center Ltd - Copyright (c) 2000-2017 Expat development team + Copyright (c) 2000 Clark Cooper <coopercc(a)users.sourceforge.net> + Copyright (c) 2002 Greg Stein <gstein(a)users.sourceforge.net> + Copyright (c) 2002 Fred L. Drake, Jr. <fdrake(a)users.sourceforge.net> + Copyright (c) 2002-2006 Karl Waclawek <karl(a)waclawek.net> + Copyright (c) 2017 Sebastian Pipping <sebastian(a)pipping.org> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining

1 0

bpo-39218: Improve accuracy of variance calculation (GH-27960)
by rhettinger Aug. 30, 2021

Aug. 30, 2021

https://github.com/python/cpython/commit/793f55bde9b0299100c12ddb0e6949c6eb… commit: 793f55bde9b0299100c12ddb0e6949c6eb4d85e5 branch: main author: Raymond Hettinger <rhettinger(a)users.noreply.github.com> committer: rhettinger <rhettinger(a)users.noreply.github.com> date: 2021-08-30T20:57:30-05:00 summary: bpo-39218: Improve accuracy of variance calculation (GH-27960) files: A Misc/NEWS.d/next/Library/2021-08-25-20-18-31.bpo-39218.BlO6jW.rst M Lib/statistics.py M Lib/test/test_statistics.py diff --git a/Lib/statistics.py b/Lib/statistics.py index 1314095332a159..a14c48e89814bd 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -728,15 +728,19 @@ def _ss(data, c=None): lead to garbage results. """ if c is not None: - T, total, count = _sum((x-c)**2 for x in data) + T, total, count = _sum((d := x - c) * d for x in data) return (T, total) + # Compute the mean accurate to within 1/2 ulp c = mean(data) - T, total, count = _sum((x-c)**2 for x in data) - # The following sum should mathematically equal zero, but due to rounding - # error may not. - U, total2, count2 = _sum((x - c) for x in data) - assert T == U and count == count2 - total -= total2 ** 2 / len(data) + # Initial computation for the sum of square deviations + T, total, count = _sum((d := x - c) * d for x in data) + # Correct any remaining inaccuracy in the mean c. + # The following sum should mathematically equal zero, + # but due to the final rounding of the mean, it may not. + U, error, count2 = _sum((x - c) for x in data) + assert count == count2 + correction = error * error / len(data) + total -= correction assert not total < 0, 'negative sum of square deviations: %f' % total return (T, total) @@ -924,8 +928,8 @@ def correlation(x, y, /): xbar = fsum(x) / n ybar = fsum(y) / n sxy = fsum((xi - xbar) * (yi - ybar) for xi, yi in zip(x, y)) - sxx = fsum((xi - xbar) ** 2.0 for xi in x) - syy = fsum((yi - ybar) ** 2.0 for yi in y) + sxx = fsum((d := xi - xbar) * d for xi in x) + syy = fsum((d := yi - ybar) * d for yi in y) try: return sxy / sqrt(sxx * syy) except ZeroDivisionError: @@ -968,7 +972,7 @@ def linear_regression(x, y, /): xbar = fsum(x) / n ybar = fsum(y) / n sxy = fsum((xi - xbar) * (yi - ybar) for xi, yi in zip(x, y)) - sxx = fsum((xi - xbar) ** 2.0 for xi in x) + sxx = fsum((d := xi - xbar) * d for xi in x) try: slope = sxy / sxx # equivalent to: covariance(x, y) / variance(x) except ZeroDivisionError: @@ -1094,10 +1098,11 @@ def samples(self, n, *, seed=None): def pdf(self, x): "Probability density function. P(x <= X < x+dx) / dx" - variance = self._sigma ** 2.0 + variance = self._sigma * self._sigma if not variance: raise StatisticsError('pdf() not defined when sigma is zero') - return exp((x - self._mu)**2.0 / (-2.0*variance)) / sqrt(tau*variance) + diff = x - self._mu + return exp(diff * diff / (-2.0 * variance)) / sqrt(tau * variance) def cdf(self, x): "Cumulative distribution function. P(X <= x)" @@ -1161,7 +1166,7 @@ def overlap(self, other): if not dv: return 1.0 - erf(dm / (2.0 * X._sigma * sqrt(2.0))) a = X._mu * Y_var - Y._mu * X_var - b = X._sigma * Y._sigma * sqrt(dm**2.0 + dv * log(Y_var / X_var)) + b = X._sigma * Y._sigma * sqrt(dm * dm + dv * log(Y_var / X_var)) x1 = (a + b) / dv x2 = (a - b) / dv return 1.0 - (fabs(Y.cdf(x1) - X.cdf(x1)) + fabs(Y.cdf(x2) - X.cdf(x2))) @@ -1204,7 +1209,7 @@ def stdev(self): @property def variance(self): "Square of the standard deviation." - return self._sigma ** 2.0 + return self._sigma * self._sigma def __add__(x1, x2): """Add a constant or another NormalDist instance. diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index dc066fa921d797..64ebd0e8cb9f84 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -1210,6 +1210,9 @@ def __pow__(self, other): def __add__(self, other): return type(self)(super().__add__(other)) __radd__ = __add__ + def __mul__(self, other): + return type(self)(super().__mul__(other)) + __rmul__ = __mul__ return (float, Decimal, Fraction, MyFloat) def test_types_conserved(self): diff --git a/Misc/NEWS.d/next/Library/2021-08-25-20-18-31.bpo-39218.BlO6jW.rst b/Misc/NEWS.d/next/Library/2021-08-25-20-18-31.bpo-39218.BlO6jW.rst new file mode 100644 index 00000000000000..7bcf9a222e33c3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-08-25-20-18-31.bpo-39218.BlO6jW.rst @@ -0,0 +1 @@ +Improve accuracy of variance calculations by using x*x instead of x**2.

1 0

bpo-45019: Add a tool to generate list of modules to include for frozen modules (gh-27980)
by ericsnowcurrently Aug. 30, 2021

Aug. 30, 2021

https://github.com/python/cpython/commit/044e8d866fdde3804bdb2282c7d23a8074… commit: 044e8d866fdde3804bdb2282c7d23a8074de8f6f branch: main author: Eric Snow <ericsnowcurrently(a)gmail.com> committer: ericsnowcurrently <ericsnowcurrently(a)gmail.com> date: 2021-08-30T17:25:11-06:00 summary: bpo-45019: Add a tool to generate list of modules to include for frozen modules (gh-27980) Frozen modules must be added to several files in order to work properly. Before this change this had to be done manually. Here we add a tool to generate the relevant lines in those files instead. This helps us avoid mistakes and omissions. https://bugs.python.org/issue45019 files: A Misc/NEWS.d/next/Build/2021-08-26-13-10-46.bpo-45019.e0mo49.rst A PCbuild/_freeze_module.vcxproj A PCbuild/_freeze_module.vcxproj.filters A Programs/_freeze_module.c A Python/frozen_modules/hello.h A Python/frozen_modules/importlib__bootstrap.h A Python/frozen_modules/importlib__bootstrap_external.h A Python/frozen_modules/zipimport.h A Tools/scripts/freeze_modules.py D PCbuild/_freeze_importlib.vcxproj D PCbuild/_freeze_importlib.vcxproj.filters D Programs/_freeze_importlib.c D Python/frozen_hello.h D Python/importlib.h D Python/importlib_external.h D Python/importlib_zipimport.h M .gitattributes M .github/workflows/build.yml M .gitignore M Doc/c-api/init.rst M Makefile.pre.in M PCbuild/pcbuild.proj M PCbuild/pcbuild.sln M PCbuild/readme.txt M Python/frozen.c M Tools/scripts/update_file.py diff --git a/.gitattributes b/.gitattributes index 68566e899249f6..b9c08cdd7d65a7 100644 --- a/.gitattributes +++ b/.gitattributes @@ -46,8 +46,7 @@ Modules/clinic/*.h linguist-generated=true Objects/clinic/*.h linguist-generated=true PC/clinic/*.h linguist-generated=true Python/clinic/*.h linguist-generated=true -Python/importlib.h linguist-generated=true -Python/importlib_external.h linguist-generated=true +Python/frozen_modules/*.h linguist-generated=true Include/internal/pycore_ast.h linguist-generated=true Python/Python-ast.c linguist-generated=true Include/opcode.h linguist-generated=true diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c5d967dec7e4b7..05bdf2445a2341 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -71,6 +71,7 @@ jobs: make regen-stdlib-module-names - name: Check for changes run: | + git add -u changes=$(git status --porcelain) # Check for changes in regenerated files if ! test -z "$changes" diff --git a/.gitignore b/.gitignore index a96be67962217e..0ed4c8bdd0ccff 100644 --- a/.gitignore +++ b/.gitignore @@ -68,7 +68,7 @@ Modules/Setup.config Modules/Setup.local Modules/config.c Modules/ld_so_aix -Programs/_freeze_importlib +Programs/_freeze_module Programs/_testembed PC/python_nt*.h PC/pythonnt_rc*.h diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 60564241bfd61e..2fcbcc8d77be45 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -110,7 +110,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Suppress error messages when calculating the module search path in :c:func:`Py_GetPath`. - Private flag used by ``_freeze_importlib`` and ``frozenmain`` programs. + Private flag used by ``_freeze_module`` and ``frozenmain`` programs. .. c:var:: int Py_HashRandomizationFlag diff --git a/Makefile.pre.in b/Makefile.pre.in index 1007f440759b1a..804d0192bc5fdb 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -574,8 +574,8 @@ coverage-lcov: @echo "lcov report at $(COVERAGE_REPORT)/index.html" @echo -# Force regeneration of parser and importlib -coverage-report: regen-token regen-importlib +# Force regeneration of parser and frozen modules +coverage-report: regen-token regen-frozen @ # build with coverage info $(MAKE) coverage @ # run tests, ignore failures @@ -734,45 +734,60 @@ Programs/_testembed: Programs/_testembed.o $(LIBRARY_DEPS) $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) ############################################################################ -# Importlib +# frozen modules (including importlib) -Programs/_freeze_importlib.o: Programs/_freeze_importlib.c Makefile +Programs/_freeze_module.o: Programs/_freeze_module.c Makefile -Programs/_freeze_importlib: Programs/_freeze_importlib.o $(LIBRARY_OBJS_OMIT_FROZEN) - $(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_importlib.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS) +Programs/_freeze_module: Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN) + $(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS) +Tools/scripts/freeze_modules.py: Programs/_freeze_module + +.PHONY: regen-frozen +regen-frozen: Tools/scripts/freeze_modules.py $(FROZEN_FILES) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/freeze_modules.py + @echo "The Makefile was updated, you may need to re-run make." + +# BEGIN: freezing modules + +Python/frozen_modules/importlib__bootstrap.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Lib/importlib/_bootstrap.py + $(srcdir)/Programs/_freeze_module importlib._bootstrap \ + $(srcdir)/Lib/importlib/_bootstrap.py \ + $(srcdir)/Python/frozen_modules/importlib__bootstrap.h + +Python/frozen_modules/importlib__bootstrap_external.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Lib/importlib/_bootstrap_external.py + $(srcdir)/Programs/_freeze_module importlib._bootstrap_external \ + $(srcdir)/Lib/importlib/_bootstrap_external.py \ + $(srcdir)/Python/frozen_modules/importlib__bootstrap_external.h + +Python/frozen_modules/zipimport.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Lib/zipimport.py + $(srcdir)/Programs/_freeze_module zipimport \ + $(srcdir)/Lib/zipimport.py \ + $(srcdir)/Python/frozen_modules/zipimport.h + +Python/frozen_modules/hello.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Tools/freeze/flag.py + $(srcdir)/Programs/_freeze_module hello \ + $(srcdir)/Tools/freeze/flag.py \ + $(srcdir)/Python/frozen_modules/hello.h + +# END: freezing modules + +# We keep this renamed target around for folks with muscle memory. .PHONY: regen-importlib -regen-importlib: Programs/_freeze_importlib - # Regenerate Python/importlib_external.h - # from Lib/importlib/_bootstrap_external.py using _freeze_importlib - ./Programs/_freeze_importlib importlib._bootstrap_external \ - $(srcdir)/Lib/importlib/_bootstrap_external.py \ - $(srcdir)/Python/importlib_external.h.new - $(UPDATE_FILE) $(srcdir)/Python/importlib_external.h $(srcdir)/Python/importlib_external.h.new - # Regenerate Python/importlib.h from Lib/importlib/_bootstrap.py - # using _freeze_importlib - ./Programs/_freeze_importlib importlib._bootstrap \ - $(srcdir)/Lib/importlib/_bootstrap.py \ - $(srcdir)/Python/importlib.h.new - $(UPDATE_FILE) $(srcdir)/Python/importlib.h $(srcdir)/Python/importlib.h.new - # Regenerate Python/importlib_zipimport.h from Lib/zipimport.py - # using _freeze_importlib - ./Programs/_freeze_importlib zipimport \ - $(srcdir)/Lib/zipimport.py \ - $(srcdir)/Python/importlib_zipimport.h.new - $(UPDATE_FILE) $(srcdir)/Python/importlib_zipimport.h $(srcdir)/Python/importlib_zipimport.h.new +regen-importlib: regen-frozen +############################################################################ +# ABI regen-limited-abi: all $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/stable_abi.py --generate-all $(srcdir)/Misc/stable_abi.txt - ############################################################################ # Regenerate all generated files regen-all: regen-opcode regen-opcode-targets regen-typeslots \ - regen-token regen-ast regen-keyword regen-importlib clinic \ - regen-pegen-metaparser regen-pegen regen-frozen regen-test-frozenmain + regen-token regen-ast regen-keyword regen-frozen clinic \ + regen-pegen-metaparser regen-pegen regen-test-frozenmain @echo @echo "Note: make regen-stdlib-module-names and autoconf should be run manually" @@ -884,15 +899,6 @@ regen-opcode: $(srcdir)/Include/opcode.h.new $(UPDATE_FILE) $(srcdir)/Include/opcode.h $(srcdir)/Include/opcode.h.new -.PHONY: regen-frozen -regen-frozen: Programs/_freeze_importlib - # Regenerate code for frozen module "__hello__". - ./Programs/_freeze_importlib hello \ - $(srcdir)/Tools/freeze/flag.py \ - $(srcdir)/Python/frozen_hello.h.new - $(UPDATE_FILE) $(srcdir)/Python/frozen_hello.h \ - $(srcdir)/Python/frozen_hello.h.new - .PHONY: regen-token regen-token: # Regenerate Doc/library/token-list.inc from Grammar/Tokens @@ -995,8 +1001,15 @@ regen-opcode-targets: Python/ceval.o: $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/ceval_gil.h \ $(srcdir)/Python/condvar.h -Python/frozen.o: $(srcdir)/Python/importlib.h $(srcdir)/Python/importlib_external.h \ - $(srcdir)/Python/importlib_zipimport.h $(srcdir)/Python/frozen_hello.h +# FROZEN_FILES is auto-generated by Tools/scripts/freeze_modules.py. +FROZEN_FILES = \ + $(srcdir)/Python/frozen_modules/importlib__bootstrap.h \ + $(srcdir)/Python/frozen_modules/importlib__bootstrap_external.h \ + $(srcdir)/Python/frozen_modules/zipimport.h \ + $(srcdir)/Python/frozen_modules/hello.h +# End FROZEN_FILES + +Python/frozen.o: $(FROZEN_FILES) # Generate DTrace probe macros, then rename them (PYTHON_ -> PyDTrace_) to # follow our naming conventions. dtrace(1) uses the output filename to generate @@ -1918,7 +1931,7 @@ clean-retain-profile: pycremoval find build -name '*.py[co]' -exec rm -f {} ';' || true -rm -f pybuilddir.txt -rm -f Lib/lib2to3/*Grammar*.pickle - -rm -f Programs/_testembed Programs/_freeze_importlib + -rm -f Programs/_testembed Programs/_freeze_module -find build -type f -a ! -name '*.gc??' -exec rm -f {} ';' -rm -f Include/pydtrace_probes.h -rm -f profile-gen-stamp diff --git a/Misc/NEWS.d/next/Build/2021-08-26-13-10-46.bpo-45019.e0mo49.rst b/Misc/NEWS.d/next/Build/2021-08-26-13-10-46.bpo-45019.e0mo49.rst new file mode 100644 index 00000000000000..d11c6451462bd5 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2021-08-26-13-10-46.bpo-45019.e0mo49.rst @@ -0,0 +1,3 @@ +Generate lines in relevant files for frozen modules. Up until now each of +the files had to be edited manually. This change makes it easier to add to +and modify the frozen modules. diff --git a/PCbuild/_freeze_importlib.vcxproj b/PCbuild/_freeze_module.vcxproj similarity index 79% rename from PCbuild/_freeze_importlib.vcxproj rename to PCbuild/_freeze_module.vcxproj index e437412a161ce5..a0bedf49e69906 100644 --- a/PCbuild/_freeze_importlib.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -69,7 +69,7 @@ <PropertyGroup Label="Globals"> <ProjectGuid>{19C0C13F-47CA-4432-AFF3-799A296A4DDC}</ProjectGuid> <Keyword>Win32Proj</Keyword> - <RootNamespace>_freeze_importlib</RootNamespace> + <RootNamespace>_freeze_module</RootNamespace> <SupportPGO>false</SupportPGO> </PropertyGroup> <Import Project="python.props" /> @@ -95,7 +95,7 @@ </Link> </ItemDefinitionGroup> <ItemGroup> - <ClCompile Include="..\Programs\_freeze_importlib.c" /> + <ClCompile Include="..\Programs\_freeze_module.c" /> </ItemGroup> <ItemGroup> <ProjectReference Include="pythoncore.vcxproj"> @@ -108,31 +108,33 @@ </ProjectReference> </ItemGroup> <ItemGroup> +  <None Include="..\Lib\importlib\_bootstrap.py"> <ModName>importlib._bootstrap</ModName> - <IntFile>$(IntDir)importlib.g.h</IntFile> - <OutFile>$(PySourcePath)Python\importlib.h</OutFile> + <IntFile>$(IntDir)importlib__bootstrap.g.h</IntFile> + <OutFile>$(PySourcePath)Python\frozen_modules\importlib__bootstrap.h</OutFile> </None> <None Include="..\Lib\importlib\_bootstrap_external.py"> <ModName>importlib._bootstrap_external</ModName> - <IntFile>$(IntDir)importlib_external.g.h</IntFile> - <OutFile>$(PySourcePath)Python\importlib_external.h</OutFile> + <IntFile>$(IntDir)importlib__bootstrap_external.g.h</IntFile> + <OutFile>$(PySourcePath)Python\frozen_modules\importlib__bootstrap_external.h</OutFile> </None> <None Include="..\Lib\zipimport.py"> <ModName>zipimport</ModName> - <IntFile>$(IntDir)importlib_zipimport.g.h</IntFile> - <OutFile>$(PySourcePath)Python\importlib_zipimport.h</OutFile> + <IntFile>$(IntDir)zipimport.g.h</IntFile> + <OutFile>$(PySourcePath)Python\frozen_modules\zipimport.h</OutFile> </None> <None Include="..\Tools\freeze\flag.py"> <ModName>hello</ModName> - <IntFile>$(IntDir)frozen_hello.g.h</IntFile> - <OutFile>$(PySourcePath)Python\frozen_hello.h</OutFile> + <IntFile>$(IntDir)ello.g.h</IntFile> + <OutFile>$(PySourcePath)Python\frozen_modules\hello.h</OutFile> </None> +  </ItemGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <ImportGroup Label="ExtensionTargets"> </ImportGroup> - <Target Name="_RebuildImportLib"> + <Target Name="_RebuildFrozen"> <Exec Command='"$(TargetPath)" "%(None.ModName)" "%(None.FullPath)" "%(None.IntFile)"' /> <Copy SourceFiles="%(None.IntFile)" @@ -143,15 +145,18 @@ <Message Text="Updated files: @(_Updated->'%(Filename)%(Extension)',', ')" Condition="'@(_Updated)' != ''" Importance="high" /> - <Warning Text="Frozen importlib files were updated. Please rebuild to pick up the changes.%0D%0A%0D%0AIf you are not developing on Windows but you see this error on a continuous integration build, please run 'make regen-all' and commit anything that changes." + <Warning Text="Frozen modules (e.g. importlib) were updated. Please rebuild to pick up the changes.%0D%0A%0D%0AIf you are not developing on Windows but you see this error on a continuous integration build, please run 'make regen-all' and commit anything that changes." Condition="'@(_Updated)' != '' and $(Configuration) == 'Debug'" /> - <Error Text="Frozen importlib files were updated. Please rebuild to pick up the changes.%0D%0A%0D%0AIf you are not developing on Windows but you see this error on a continuous integration build, please run 'make regen-all' and commit anything that changes." + <Error Text="Frozen (e.g. importlib) files were updated. Please rebuild to pick up the changes.%0D%0A%0D%0AIf you are not developing on Windows but you see this error on a continuous integration build, please run 'make regen-all' and commit anything that changes." Condition="'@(_Updated)' != '' and $(Configuration) == 'Release'" /> </Target> + <Target Name="RebuildFrozen" AfterTargets="AfterBuild" Condition="$(Configuration) == 'Debug' or $(Configuration) == 'Release'" + DependsOnTargets="_RebuildFrozen"> + </Target> <Target Name="RebuildImportLib" AfterTargets="AfterBuild" Condition="$(Configuration) == 'Debug' or $(Configuration) == 'Release'" - DependsOnTargets="_RebuildImportLib"> + DependsOnTargets="_RebuildFrozen"> </Target> - <Target Name="_CleanImportLib" BeforeTargets="CoreClean"> + <Target Name="_CleanFrozen" BeforeTargets="CoreClean"> <ItemGroup> <Clean Include="%(None.IntFile)" /> </ItemGroup> diff --git a/PCbuild/_freeze_importlib.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters similarity index 76% rename from PCbuild/_freeze_importlib.vcxproj.filters rename to PCbuild/_freeze_module.vcxproj.filters index 3ee9eb750d67e8..bed7920fdba638 100644 --- a/PCbuild/_freeze_importlib.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -10,19 +10,24 @@ </Filter> </ItemGroup> <ItemGroup> - <ClCompile Include="..\Programs\_freeze_importlib.c"> + <ClCompile Include="..\Programs\_freeze_module.c"> <Filter>Source Files</Filter> </ClCompile> </ItemGroup> <ItemGroup> +  <None Include="..\Lib\importlib\_bootstrap.py"> - <Filter>Source Files</Filter> + <Filter>Python Files</Filter> + </None> + <None Include="..\Lib\importlib\_bootstrap_external.py"> + <Filter>Python Files</Filter> </None> <None Include="..\Lib\zipimport.py"> <Filter>Python Files</Filter> </None> - <None Include="..\Lib\importlib\_bootstrap_external.py"> + <None Include="..\Tools\freeze\flag.py"> <Filter>Python Files</Filter> </None> +  </ItemGroup> -</Project> \ No newline at end of file +</Project> diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj index 8e7088d47d2aed..f464ad3b18e44c 100644 --- a/PCbuild/pcbuild.proj +++ b/PCbuild/pcbuild.proj @@ -72,8 +72,8 @@ <BuildInParallel>false</BuildInParallel> </Projects> -  - <Projects2 Condition="$(Platform) != 'ARM' and $(Platform) != 'ARM64'" Include="_freeze_importlib.vcxproj" /> +  + <Projects2 Condition="$(Platform) != 'ARM' and $(Platform) != 'ARM64'" Include="_freeze_module.vcxproj" />  <Projects2 Include="python.vcxproj;pythonw.vcxproj" /> <Projects2 Include="python_uwp.vcxproj;pythonw_uwp.vcxproj" Condition="$(IncludeUwp)" /> diff --git a/PCbuild/pcbuild.sln b/PCbuild/pcbuild.sln index 3507b972797c96..c774e049717352 100644 --- a/PCbuild/pcbuild.sln +++ b/PCbuild/pcbuild.sln @@ -75,7 +75,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pywlauncher", "pywlauncher. {7B2727B5-5A3F-40EE-A866-43A13CD31446} = {7B2727B5-5A3F-40EE-A866-43A13CD31446} EndProjectSection EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_freeze_importlib", "_freeze_importlib.vcxproj", "{19C0C13F-47CA-4432-AFF3-799A296A4DDC}" +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_freeze_module", "_freeze_module.vcxproj", "{19C0C13F-47CA-4432-AFF3-799A296A4DDC}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_overlapped", "_overlapped.vcxproj", "{EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}" EndProject diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 6c25522ea48c00..5ecded06e58893 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -115,9 +115,10 @@ _testembed These are miscellaneous sub-projects that don't really fit the other categories: -_freeze_importlib - _freeze_importlib.exe, used to regenerate Python\importlib.h after - changes have been made to Lib\importlib\_bootstrap.py +_freeze_module + _freeze_module.exe, used to regenerate frozen modules in Python + after changes have been made to the corresponding source files + (e.g. Lib\importlib\_bootstrap.py). pyshellext pyshellext.dll, the shell extension deployed with the launcher python3dll diff --git a/Programs/_freeze_importlib.c b/Programs/_freeze_module.c similarity index 53% rename from Programs/_freeze_importlib.c rename to Programs/_freeze_module.c index 2e4ccbb154a414..7e9f02aec8a0fa 100644 --- a/Programs/_freeze_importlib.c +++ b/Programs/_freeze_module.c @@ -1,5 +1,10 @@ /* This is built as a stand-alone executable by the Makefile, and helps turn - Lib/importlib/_bootstrap.py into a frozen module in Python/importlib.h + modules into frozen modules (like Lib/importlib/_bootstrap.py + into Python/importlib.h). + + This is used directly by Tools/scripts/freeze_modules.py, and indirectly by "make regen-frozen". + + See Python/frozen.c for more info. */ #include <Python.h> @@ -28,54 +33,11 @@ const struct _frozen *PyImport_FrozenModules; #endif static const char header[] = - "/* Auto-generated by Programs/_freeze_importlib.c */"; + "/* Auto-generated by Programs/_freeze_module.c */"; -int -main(int argc, char *argv[]) +static void +runtime_init(void) { - const char *name, *inpath, *outpath; - char buf[100]; - FILE *infile = NULL, *outfile = NULL; - struct _Py_stat_struct stat; - size_t text_size, data_size, i, n; - char *text = NULL; - unsigned char *data; - PyObject *code = NULL, *marshalled = NULL; - - PyImport_FrozenModules = _PyImport_FrozenModules; - - if (argc != 4) { - fprintf(stderr, "need to specify the name, input and output paths\n"); - return 2; - } - name = argv[1]; - inpath = argv[2]; - outpath = argv[3]; - infile = fopen(inpath, "rb"); - if (infile == NULL) { - fprintf(stderr, "cannot open '%s' for reading\n", inpath); - goto error; - } - if (_Py_fstat_noraise(fileno(infile), &stat)) { - fprintf(stderr, "cannot fstat '%s'\n", inpath); - goto error; - } - text_size = (size_t)stat.st_size; - text = (char *) malloc(text_size + 1); - if (text == NULL) { - fprintf(stderr, "could not allocate %ld bytes\n", (long) text_size); - goto error; - } - n = fread(text, 1, text_size, infile); - fclose(infile); - infile = NULL; - if (n < text_size) { - fprintf(stderr, "read too short: got %ld instead of %ld bytes\n", - (long) n, (long) text_size); - goto error; - } - text[text_size] = '\0'; - PyConfig config; PyConfig_InitIsolatedConfig(&config); @@ -83,7 +45,7 @@ main(int argc, char *argv[]) PyStatus status; status = PyConfig_SetString(&config, &config.program_name, - L"./_freeze_importlib"); + L"./_freeze_module"); if (PyStatus_Exception(status)) { PyConfig_Clear(&config); Py_ExitStatusException(status); @@ -98,39 +60,93 @@ main(int argc, char *argv[]) if (PyStatus_Exception(status)) { Py_ExitStatusException(status); } +} - sprintf(buf, "<frozen %s>", name); - code = Py_CompileStringExFlags(text, buf, Py_file_input, NULL, 0); - if (code == NULL) - goto error; - free(text); - text = NULL; +static const char * +read_text(const char *inpath) +{ + FILE *infile = fopen(inpath, "rb"); + if (infile == NULL) { + fprintf(stderr, "cannot open '%s' for reading\n", inpath); + return NULL; + } - marshalled = PyMarshal_WriteObjectToString(code, Py_MARSHAL_VERSION); - Py_CLEAR(code); - if (marshalled == NULL) - goto error; + struct _Py_stat_struct stat; + if (_Py_fstat_noraise(fileno(infile), &stat)) { + fprintf(stderr, "cannot fstat '%s'\n", inpath); + fclose(infile); + return NULL; + } + size_t text_size = (size_t)stat.st_size; - assert(PyBytes_CheckExact(marshalled)); - data = (unsigned char *) PyBytes_AS_STRING(marshalled); - data_size = PyBytes_GET_SIZE(marshalled); + char *text = (char *) malloc(text_size + 1); + if (text == NULL) { + fprintf(stderr, "could not allocate %ld bytes\n", (long) text_size); + fclose(infile); + return NULL; + } + size_t n = fread(text, 1, text_size, infile); + fclose(infile); - /* Open the file in text mode. The hg checkout should be using the eol extension, - which in turn should cause the EOL style match the C library's text mode */ - outfile = fopen(outpath, "w"); - if (outfile == NULL) { - fprintf(stderr, "cannot open '%s' for writing\n", outpath); - goto error; + if (n < text_size) { + fprintf(stderr, "read too short: got %ld instead of %ld bytes\n", + (long) n, (long) text_size); + free(text); + return NULL; } - fprintf(outfile, "%s\n", header); - for (i = n = 0; name[i] != '\0'; i++) { - if (name[i] != '.') { - buf[n++] = name[i]; + + text[text_size] = '\0'; + return (const char *)text; +} + +static PyObject * +compile_and_marshal(const char *name, const char *text) +{ + char *filename = (char *) malloc(strlen(name) + 10); + sprintf(filename, "<frozen %s>", name); + PyObject *code = Py_CompileStringExFlags(text, filename, + Py_file_input, NULL, 0); + free(filename); + if (code == NULL) { + return NULL; + } + + PyObject *marshalled = PyMarshal_WriteObjectToString(code, Py_MARSHAL_VERSION); + Py_CLEAR(code); + if (marshalled == NULL) { + return NULL; + } + assert(PyBytes_CheckExact(marshalled)); + + return marshalled; +} + +static char * +get_varname(const char *name, const char *prefix) +{ + size_t n = strlen(prefix); + char *varname = (char *) malloc(strlen(name) + n + 1); + (void)strcpy(varname, prefix); + for (size_t i = 0; name[i] != '\0'; i++) { + if (name[i] == '.') { + varname[n++] = '_'; + } + else { + varname[n++] = name[i]; } } - buf[n] = '\0'; - fprintf(outfile, "const unsigned char _Py_M__%s[] = {\n", buf); - for (n = 0; n < data_size; n += 16) { + varname[n] = '\0'; + return varname; +} + +static void +write_code(FILE *outfile, PyObject *marshalled, const char *varname) +{ + unsigned char *data = (unsigned char *) PyBytes_AS_STRING(marshalled); + size_t data_size = PyBytes_GET_SIZE(marshalled); + + fprintf(outfile, "const unsigned char %s[] = {\n", varname); + for (size_t n = 0; n < data_size; n += 16) { size_t i, end = Py_MIN(n + 16, data_size); fprintf(outfile, " "); for (i = n; i < end; i++) { @@ -139,29 +155,72 @@ main(int argc, char *argv[]) fprintf(outfile, "\n"); } fprintf(outfile, "};\n"); +} + +static int +write_frozen(const char *outpath, const char *inpath, const char *name, + PyObject *marshalled) +{ + /* Open the file in text mode. The hg checkout should be using the eol extension, + which in turn should cause the EOL style match the C library's text mode */ + FILE *outfile = fopen(outpath, "w"); + if (outfile == NULL) { + fprintf(stderr, "cannot open '%s' for writing\n", outpath); + return -1; + } - Py_CLEAR(marshalled); + fprintf(outfile, "%s\n", header); + char *arrayname = get_varname(name, "_Py_M__"); + write_code(outfile, marshalled, arrayname); + free(arrayname); - Py_Finalize(); - if (outfile) { - if (ferror(outfile)) { - fprintf(stderr, "error when writing to '%s'\n", outpath); - goto error; - } - fclose(outfile); + if (ferror(outfile)) { + fprintf(stderr, "error when writing to '%s'\n", outpath); + return -1; } + fclose(outfile); + return 0; +} + +int +main(int argc, char *argv[]) +{ + const char *name, *inpath, *outpath; + + PyImport_FrozenModules = _PyImport_FrozenModules; + + if (argc != 4) { + fprintf(stderr, "need to specify the name, input and output paths\n"); + return 2; + } + name = argv[1]; + inpath = argv[2]; + outpath = argv[3]; + + runtime_init(); + + const char *text = read_text(inpath); + if (text == NULL) { + goto error; + } + + PyObject *marshalled = compile_and_marshal(name, text); + free((char *)text); + if (marshalled == NULL) { + goto error; + } + + int res = write_frozen(outpath, inpath, name, marshalled); + Py_DECREF(marshalled); + if (res != 0) { + goto error; + } + + Py_Finalize(); return 0; error: PyErr_Print(); Py_Finalize(); - if (infile) - fclose(infile); - if (outfile) - fclose(outfile); - if (text) - free(text); - if (marshalled) - Py_DECREF(marshalled); return 1; } diff --git a/Python/frozen.c b/Python/frozen.c index 7f433ff80ca129..67aff2ed2eba14 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -1,35 +1,63 @@ -/* Frozen modules initializer */ - -#include "Python.h" -#include "importlib.h" -#include "importlib_external.h" -#include "importlib_zipimport.h" +/* Frozen modules initializer + * + * Frozen modules are written to header files by Programs/_freeze_module. + * These files are typically put in Python/frozen_modules/. Each holds + * an array of bytes named "_Py_M__<module>", which is used below. + * + * These files must be regenerated any time the corresponding .pyc + * file would change (including with changes to the compiler, bytecode + * format, marshal format). This can be done with "make regen-frozen". + * That make target just runs Tools/scripts/freeze_modules.py. + * + * The freeze_modules.py script also determines which modules get + * frozen. Update the list at the top of the script to add, remove, + * or modify the target modules. Then run the script + * (or run "make regen-frozen"). + * + * The script does the following: + * + * 1. run Programs/_freeze_module on the target modules + * 2. update the includes and _PyImport_FrozenModules[] in this file + * 3. update the FROZEN_FILES variable in Makefile.pre.in + * 4. update the per-module targets in Makefile.pre.in + * 5. update the lists of modules in PCbuild/_freeze_module.vcxproj and + * PCbuild/_freeze_module.vcxproj.filters + * + * (Note that most of the data in this file is auto-generated by the script.) + * + * Those steps can also be done manually, though this is not recommended. + * Expect such manual changes to be removed the next time + * freeze_modules.py runs. + * */ /* In order to test the support for frozen modules, by default we - define a single frozen module, __hello__. Loading it will print - some famous words... */ + define some simple frozen modules: __hello__, __phello__ (a package), + and __phello__.spam. Loading any will print some famous words... */ -/* Run "make regen-frozen" to regen the file below (e.g. after a bytecode - * format change). The include file defines _Py_M__hello as an array of bytes. - */ -#include "frozen_hello.h" +#include "Python.h" -#define SIZE (int)sizeof(_Py_M__hello) +/* Includes for frozen modules: */ +#include "frozen_modules/importlib__bootstrap.h" +#include "frozen_modules/importlib__bootstrap_external.h" +#include "frozen_modules/zipimport.h" +#include "frozen_modules/hello.h" +/* End includes */ + +/* Note that a negative size indicates a package. */ static const struct _frozen _PyImport_FrozenModules[] = { /* importlib */ - {"_frozen_importlib", _Py_M__importlib_bootstrap, - (int)sizeof(_Py_M__importlib_bootstrap)}, - {"_frozen_importlib_external", _Py_M__importlib_bootstrap_external, - (int)sizeof(_Py_M__importlib_bootstrap_external)}, - {"zipimport", _Py_M__zipimport, - (int)sizeof(_Py_M__zipimport)}, + {"_frozen_importlib", _Py_M__importlib__bootstrap, + (int)sizeof(_Py_M__importlib__bootstrap)}, + {"_frozen_importlib_external", _Py_M__importlib__bootstrap_external, + (int)sizeof(_Py_M__importlib__bootstrap_external)}, + {"zipimport", _Py_M__zipimport, (int)sizeof(_Py_M__zipimport)}, + /* Test module */ - {"__hello__", _Py_M__hello, SIZE}, - /* Test package (negative size indicates package-ness) */ - {"__phello__", _Py_M__hello, -SIZE}, - {"__phello__.spam", _Py_M__hello, SIZE}, + {"__hello__", _Py_M__hello, (int)sizeof(_Py_M__hello)}, + {"__phello__", _Py_M__hello, -(int)sizeof(_Py_M__hello)}, + {"__phello__.spam", _Py_M__hello, (int)sizeof(_Py_M__hello)}, {0, 0, 0} /* sentinel */ }; diff --git a/Python/frozen_hello.h b/Python/frozen_modules/hello.h similarity index 91% rename from Python/frozen_hello.h rename to Python/frozen_modules/hello.h index c65c661e9bfb67..2658c05886a6db 100644 --- a/Python/frozen_hello.h +++ b/Python/frozen_modules/hello.h @@ -1,4 +1,4 @@ -/* Auto-generated by Programs/_freeze_importlib.c */ +/* Auto-generated by Programs/_freeze_module.c */ const unsigned char _Py_M__hello[] = { 99,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0, 0,0,0,0,0,115,16,0,0,0,100,0,90,0,101,1, diff --git a/Python/importlib.h b/Python/frozen_modules/importlib__bootstrap.h similarity index 99% rename from Python/importlib.h rename to Python/frozen_modules/importlib__bootstrap.h index 69bd9727237f17..2716896c21f4a5 100644 --- a/Python/importlib.h +++ b/Python/frozen_modules/importlib__bootstrap.h @@ -1,5 +1,5 @@ -/* Auto-generated by Programs/_freeze_importlib.c */ -const unsigned char _Py_M__importlib_bootstrap[] = { +/* Auto-generated by Programs/_freeze_module.c */ +const unsigned char _Py_M__importlib__bootstrap[] = { 99,0,0,0,0,0,0,0,0,0,0,0,0,4,0,0, 0,0,0,0,0,115,130,1,0,0,100,0,90,0,100,1, 132,0,90,1,100,2,90,2,100,2,90,3,100,2,90,4, diff --git a/Python/importlib_external.h b/Python/frozen_modules/importlib__bootstrap_external.h similarity index 99% rename from Python/importlib_external.h rename to Python/frozen_modules/importlib__bootstrap_external.h index c49fa5516eb26c..7a3410067d4a80 100644 --- a/Python/importlib_external.h +++ b/Python/frozen_modules/importlib__bootstrap_external.h @@ -1,5 +1,5 @@ -/* Auto-generated by Programs/_freeze_importlib.c */ -const unsigned char _Py_M__importlib_bootstrap_external[] = { +/* Auto-generated by Programs/_freeze_module.c */ +const unsigned char _Py_M__importlib__bootstrap_external[] = { 99,0,0,0,0,0,0,0,0,0,0,0,0,5,0,0, 0,0,0,0,0,115,158,2,0,0,100,0,90,0,100,1, 97,1,100,2,100,1,108,2,90,2,100,2,100,1,108,3, diff --git a/Python/importlib_zipimport.h b/Python/frozen_modules/zipimport.h similarity index 99% rename from Python/importlib_zipimport.h rename to Python/frozen_modules/zipimport.h index c12ed5215b3f88..b4e2e85283cf49 100644 --- a/Python/importlib_zipimport.h +++ b/Python/frozen_modules/zipimport.h @@ -1,4 +1,4 @@ -/* Auto-generated by Programs/_freeze_importlib.c */ +/* Auto-generated by Programs/_freeze_module.c */ const unsigned char _Py_M__zipimport[] = { 99,0,0,0,0,0,0,0,0,0,0,0,0,4,0,0, 0,0,0,0,0,115,48,1,0,0,100,0,90,0,100,1, diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py new file mode 100644 index 00000000000000..4f60e1b9a3a8ba --- /dev/null +++ b/Tools/scripts/freeze_modules.py @@ -0,0 +1,496 @@ +"""Freeze modules and regen related files (e.g. Python/frozen.c). + +See the notes at the top of Python/frozen.c for more info. +""" + +import os +import os.path +import subprocess +import sys +import textwrap + +from update_file import updating_file_with_tmpfile + + +SCRIPTS_DIR = os.path.abspath(os.path.dirname(__file__)) +TOOLS_DIR = os.path.dirname(SCRIPTS_DIR) +ROOT_DIR = os.path.dirname(TOOLS_DIR) + +STDLIB_DIR = os.path.join(ROOT_DIR, 'Lib') +# If MODULES_DIR is changed then the .gitattributes file needs to be updated. +MODULES_DIR = os.path.join(ROOT_DIR, 'Python/frozen_modules') +TOOL = os.path.join(ROOT_DIR, 'Programs', '_freeze_module') + +FROZEN_FILE = os.path.join(ROOT_DIR, 'Python', 'frozen.c') +MAKEFILE = os.path.join(ROOT_DIR, 'Makefile.pre.in') +PCBUILD_PROJECT = os.path.join(ROOT_DIR, 'PCbuild', '_freeze_module.vcxproj') +PCBUILD_FILTERS = os.path.join(ROOT_DIR, 'PCbuild', '_freeze_module.vcxproj.filters') + +# These are modules that get frozen. +FROZEN = [ + # See parse_frozen_spec() for the format. + # In cases where the frozenid is duplicated, the first one is re-used. + ('importlib', [ + 'importlib._bootstrap : _frozen_importlib', + 'importlib._bootstrap_external : _frozen_importlib_external', + 'zipimport', + ]), + ('Test module', [ + 'hello : __hello__ = ' + os.path.join(TOOLS_DIR, 'freeze', 'flag.py'), + 'hello : <__phello__>', + 'hello : __phello__.spam', + ]), +] + + +####################################### +# specs + +def parse_frozen_spec(rawspec, knownids=None, section=None): + """Yield (frozenid, pyfile, modname, ispkg) for the corresponding modules. + + Supported formats: + + frozenid + frozenid : modname + frozenid : modname = pyfile + + "frozenid" and "modname" must be valid module names (dot-separated + identifiers). If "modname" is not provided then "frozenid" is used. + If "pyfile" is not provided then the filename of the module + corresponding to "frozenid" is used. + + Angle brackets around a frozenid (e.g. '<encodings>") indicate + it is a package. This also means it must be an actual module + (i.e. "pyfile" cannot have been provided). Such values can have + patterns to expand submodules: + + <encodings.*> - also freeze all direct submodules + <encodings.**.*> - also freeze the full submodule tree + + As with "frozenid", angle brackets around "modname" indicate + it is a package. However, in this case "pyfile" should not + have been provided and patterns in "modname" are not supported. + Also, if "modname" has brackets then "frozenid" should not, + and "pyfile" should have been provided.. + """ + frozenid, _, remainder = rawspec.partition(':') + modname, _, pyfile = remainder.partition('=') + frozenid = frozenid.strip() + modname = modname.strip() + pyfile = pyfile.strip() + + submodules = None + if modname.startswith('<') and modname.endswith('>'): + assert check_modname(frozenid), rawspec + modname = modname[1:-1] + assert check_modname(modname), rawspec + if frozenid in knownids: + pass + elif pyfile: + assert not os.path.isdir(pyfile), rawspec + else: + pyfile = _resolve_module(frozenid, ispkg=False) + ispkg = True + elif pyfile: + assert check_modname(frozenid), rawspec + assert not knownids or frozenid not in knownids, rawspec + assert check_modname(modname), rawspec + assert not os.path.isdir(pyfile), rawspec + ispkg = False + elif knownids and frozenid in knownids: + assert check_modname(frozenid), rawspec + assert check_modname(modname), rawspec + ispkg = False + else: + assert not modname or check_modname(modname), rawspec + resolved = iter(resolve_modules(frozenid)) + frozenid, pyfile, ispkg = next(resolved) + if not modname: + modname = frozenid + if ispkg: + pkgid = frozenid + pkgname = modname + def iter_subs(): + for frozenid, pyfile, ispkg in resolved: + assert not knownids or frozenid not in knownids, (frozenid, rawspec) + if pkgname: + modname = frozenid.replace(pkgid, pkgname, 1) + else: + modname = frozenid + yield frozenid, pyfile, modname, ispkg, section + submodules = iter_subs() + + spec = (frozenid, pyfile or None, modname, ispkg, section) + return spec, submodules + + +def parse_frozen_specs(rawspecs=FROZEN): + seen = set() + for section, _specs in rawspecs: + for spec in _parse_frozen_specs(_specs, section, seen): + frozenid = spec[0] + yield spec + seen.add(frozenid) + + +def _parse_frozen_specs(rawspecs, section, seen): + for rawspec in rawspecs: + spec, subs = parse_frozen_spec(rawspec, seen, section) + yield spec + for spec in subs or (): + yield spec + + +def resolve_frozen_file(spec, destdir=MODULES_DIR): + if isinstance(spec, str): + modname = spec + else: + _, frozenid, _, _, _= spec + modname = frozenid + # We use a consistent naming convention for all frozen modules. + return os.path.join(destdir, modname.replace('.', '_')) + '.h' + + +def resolve_frozen_files(specs, destdir=MODULES_DIR): + frozen = {} + frozenids = [] + lastsection = None + for spec in specs: + frozenid, pyfile, *_, section = spec + if frozenid in frozen: + if section is None: + lastsection = None + else: + assert section == lastsection + continue + lastsection = section + frozenfile = resolve_frozen_file(frozenid, destdir) + frozen[frozenid] = (pyfile, frozenfile) + frozenids.append(frozenid) + return frozen, frozenids + + +####################################### +# generic helpers + +def resolve_modules(modname, pyfile=None): + if modname.startswith('<') and modname.endswith('>'): + if pyfile: + assert os.path.isdir(pyfile) or os.path.basename(pyfile) == '__init__.py', pyfile + ispkg = True + modname = modname[1:-1] + rawname = modname + # For now, we only expect match patterns at the end of the name. + _modname, sep, match = modname.rpartition('.') + if sep: + if _modname.endswith('.**'): + modname = _modname[:-3] + match = f'**.{match}' + elif match and not match.isidentifier(): + modname = _modname + # Otherwise it's a plain name so we leave it alone. + else: + match = None + else: + ispkg = False + rawname = modname + match = None + + if not check_modname(modname): + raise ValueError(f'not a valid module name ({rawname})') + + if not pyfile: + pyfile = _resolve_module(modname, ispkg=ispkg) + elif os.path.isdir(pyfile): + pyfile = _resolve_module(modname, pyfile, ispkg) + yield modname, pyfile, ispkg + + if match: + pkgdir = os.path.dirname(pyfile) + yield from iter_submodules(modname, pkgdir, match) + + +def check_modname(modname): + return all(n.isidentifier() for n in modname.split('.')) + + +def iter_submodules(pkgname, pkgdir=None, match='*'): + if not pkgdir: + pkgdir = os.path.join(STDLIB_DIR, *pkgname.split('.')) + if not match: + match = '**.*' + match_modname = _resolve_modname_matcher(match, pkgdir) + + def _iter_submodules(pkgname, pkgdir): + for entry in sorted(os.scandir(pkgdir), key=lambda e: e.name): + matched, recursive = match_modname(entry.name) + if not matched: + continue + modname = f'{pkgname}.{entry.name}' + if modname.endswith('.py'): + yield modname[:-3], entry.path, False + elif entry.is_dir(): + pyfile = os.path.join(entry.path, '__init__.py') + # We ignore namespace packages. + if os.path.exists(pyfile): + yield modname, pyfile, True + if recursive: + yield from _iter_submodules(modname, entry.path) + + return _iter_submodules(pkgname, pkgdir) + + +def _resolve_modname_matcher(match, rootdir=None): + if isinstance(match, str): + if match.startswith('**.'): + recursive = True + pat = match[3:] + assert match + else: + recursive = False + pat = match + + if pat == '*': + def match_modname(modname): + return True, recursive + else: + raise NotImplementedError(match) + elif callable(match): + match_modname = match(rootdir) + else: + raise ValueError(f'unsupported matcher {match!r}') + return match_modname + + +def _resolve_module(modname, pathentry=STDLIB_DIR, ispkg=False): + assert pathentry, pathentry + pathentry = os.path.normpath(pathentry) + assert os.path.isabs(pathentry) + if ispkg: + return os.path.join(pathentry, *modname.split('.'), '__init__.py') + return os.path.join(pathentry, *modname.split('.')) + '.py' + + +####################################### +# regenerating dependent files + +def find_marker(lines, marker, file): + for pos, line in enumerate(lines): + if marker in line: + return pos + raise Exception(f"Can't find {marker!r} in file {file}") + + +def replace_block(lines, start_marker, end_marker, replacements, file): + start_pos = find_marker(lines, start_marker, file) + end_pos = find_marker(lines, end_marker, file) + if end_pos <= start_pos: + raise Exception(f"End marker {end_marker!r} " + f"occurs before start marker {start_marker!r} " + f"in file {file}") + replacements = [line.rstrip() + os.linesep for line in replacements] + return lines[:start_pos + 1] + replacements + lines[end_pos:] + + +def regen_frozen(specs, dest=MODULES_DIR): + if isinstance(dest, str): + frozen, frozenids = resolve_frozen_files(specs, destdir) + else: + frozenids, frozen = dest + + headerlines = [] + parentdir = os.path.dirname(FROZEN_FILE) + for frozenid in frozenids: + # Adding a comment to separate sections here doesn't add much, + # so we don't. + _, frozenfile = frozen[frozenid] + header = os.path.relpath(frozenfile, parentdir) + headerlines.append(f'#include "{header}"') + + deflines = [] + indent = ' ' + lastsection = None + for spec in specs: + frozenid, _, modname, ispkg, section = spec + if section != lastsection: + if lastsection is not None: + deflines.append('') + deflines.append(f'/* {section} */') + lastsection = section + + # This matches what we do in Programs/_freeze_module.c: + name = frozenid.replace('.', '_') + symbol = '_Py_M__' + name + pkg = '-' if ispkg else '' + line = ('{"%s", %s, %s(int)sizeof(%s)},' + % (modname, symbol, pkg, symbol)) + # TODO: Consider not folding lines + if len(line) < 80: + deflines.append(line) + else: + line1, _, line2 = line.rpartition(' ') + deflines.append(line1) + deflines.append(indent + line2) + + if not deflines[0]: + del deflines[0] + for i, line in enumerate(deflines): + if line: + deflines[i] = indent + line + + print(f'# Updating {os.path.relpath(FROZEN_FILE)}') + with updating_file_with_tmpfile(FROZEN_FILE) as (infile, outfile): + lines = infile.readlines() + # TODO: Use more obvious markers, e.g. + # $START GENERATED FOOBAR$ / $END GENERATED FOOBAR$ + lines = replace_block( + lines, + "/* Includes for frozen modules: */", + "/* End includes */", + headerlines, + FROZEN_FILE, + ) + lines = replace_block( + lines, + "static const struct _frozen _PyImport_FrozenModules[] =", + "/* sentinel */", + deflines, + FROZEN_FILE, + ) + outfile.writelines(lines) + + +def regen_makefile(frozenids, frozen): + frozenfiles = [] + rules = [''] + for frozenid in frozenids: + pyfile, frozenfile = frozen[frozenid] + header = os.path.relpath(frozenfile, ROOT_DIR) + relfile = header.replace('\\', '/') + frozenfiles.append(f'\t\t$(srcdir)/{relfile} \\') + + _pyfile = os.path.relpath(pyfile, ROOT_DIR) + tmpfile = f'{header}.new' + # Note that we freeze the module to the target .h file + # instead of going through an intermediate file like we used to. + rules.append(f'{header}: $(srcdir)/Programs/_freeze_module $(srcdir)/{_pyfile}') + rules.append(f'\t$(srcdir)/Programs/_freeze_module {frozenid} \\') + rules.append(f'\t\t$(srcdir)/{_pyfile} \\') + rules.append(f'\t\t$(srcdir)/{header}') + rules.append('') + + frozenfiles[-1] = frozenfiles[-1].rstrip(" \\") + + print(f'# Updating {os.path.relpath(MAKEFILE)}') + with updating_file_with_tmpfile(MAKEFILE) as (infile, outfile): + lines = infile.readlines() + lines = replace_block( + lines, + "FROZEN_FILES =", + "# End FROZEN_FILES", + frozenfiles, + MAKEFILE, + ) + lines = replace_block( + lines, + "# BEGIN: freezing modules", + "# END: freezing modules", + rules, + MAKEFILE, + ) + outfile.writelines(lines) + + +def regen_pcbuild(frozenids, frozen): + projlines = [] + filterlines = [] + for frozenid in frozenids: + pyfile, frozenfile = frozen[frozenid] + + _pyfile = os.path.relpath(pyfile, ROOT_DIR).replace('/', '\\') + header = os.path.relpath(frozenfile, ROOT_DIR).replace('/', '\\') + intfile = header.split('\\')[-1].strip('.h') + '.g.h' + projlines.append(f' <None Include="..\\{_pyfile}">') + projlines.append(f' <ModName>{frozenid}</ModName>') + projlines.append(f' <IntFile>$(IntDir){intfile}</IntFile>') + projlines.append(f' <OutFile>$(PySourcePath){header}</OutFile>') + projlines.append(f' </None>') + + filterlines.append(f' <None Include="..\\{_pyfile}">') + filterlines.append(' <Filter>Python Files</Filter>') + filterlines.append(' </None>') + + print(f'# Updating {os.path.relpath(PCBUILD_PROJECT)}') + with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile): + lines = infile.readlines() + lines = replace_block( + lines, + '', + '', + projlines, + PCBUILD_PROJECT, + ) + outfile.writelines(lines) + print(f'# Updating {os.path.relpath(PCBUILD_FILTERS)}') + with updating_file_with_tmpfile(PCBUILD_FILTERS) as (infile, outfile): + lines = infile.readlines() + lines = replace_block( + lines, + '', + '', + filterlines, + PCBUILD_FILTERS, + ) + outfile.writelines(lines) + + +####################################### +# freezing modules + +def freeze_module(modname, pyfile=None, destdir=MODULES_DIR): + """Generate the frozen module .h file for the given module.""" + for modname, pyfile, ispkg in resolve_modules(modname, pyfile): + frozenfile = _resolve_frozen(modname, destdir) + _freeze_module(modname, pyfile, frozenfile) + + +def _freeze_module(frozenid, pyfile, frozenfile): + tmpfile = frozenfile + '.new' + + argv = [TOOL, frozenid, pyfile, tmpfile] + print('#', ' '.join(os.path.relpath(a) for a in argv)) + try: + subprocess.run(argv, check=True) + except subprocess.CalledProcessError: + if not os.path.exists(TOOL): + sys.exit(f'ERROR: missing {TOOL}; you need to run "make regen-frozen"') + raise # re-raise + + os.replace(tmpfile, frozenfile) + + +####################################### +# the script + +def main(): + # Expand the raw specs, preserving order. + specs = list(parse_frozen_specs()) + frozen, frozenids = resolve_frozen_files(specs, MODULES_DIR) + + # Regen build-related files. + regen_frozen(specs, (frozenids, frozen)) + regen_makefile(frozenids, frozen) + regen_pcbuild(frozenids, frozen) + + # Freeze the target modules. + for frozenid in frozenids: + pyfile, frozenfile = frozen[frozenid] + _freeze_module(frozenid, pyfile, frozenfile) + + +if __name__ == '__main__': + argv = sys.argv[1:] + if argv: + sys.exit('ERROR: got unexpected args {argv}') + main() diff --git a/Tools/scripts/update_file.py b/Tools/scripts/update_file.py index 224585c69bbaeb..cfc4e2b1ab12a3 100644 --- a/Tools/scripts/update_file.py +++ b/Tools/scripts/update_file.py @@ -6,23 +6,47 @@ actually change the in-tree generated code. """ +import contextlib import os +import os.path import sys -def main(old_path, new_path): - with open(old_path, 'rb') as f: +(a)contextlib.contextmanager +def updating_file_with_tmpfile(filename, tmpfile=None): + """A context manager for updating a file via a temp file. + + The context manager provides two open files: the source file open + for reading, and the temp file, open for writing. + + Upon exiting: both files are closed, and the source file is replaced + with the temp file. + """ + # XXX Optionally use tempfile.TemporaryFile? + if not tmpfile: + tmpfile = filename + '.tmp' + elif os.path.isdir(tmpfile): + tmpfile = os.path.join(tmpfile, filename + '.tmp') + + with open(tmpfile, 'w') as outfile: + with open(filename) as infile: + yield infile, outfile + update_file_with_tmpfile(filename, tmpfile) + + +def update_file_with_tmpfile(filename, tmpfile): + with open(filename, 'rb') as f: old_contents = f.read() - with open(new_path, 'rb') as f: + with open(tmpfile, 'rb') as f: new_contents = f.read() if old_contents != new_contents: - os.replace(new_path, old_path) + os.replace(tmpfile, filename) else: - os.unlink(new_path) + os.unlink(tmpfile) if __name__ == '__main__': if len(sys.argv) != 3: print("Usage: %s <path to be updated> <path with new contents>" % (sys.argv[0],)) sys.exit(1) - main(sys.argv[1], sys.argv[2]) + update_file_with_tmpfile(sys.argv[1], sys.argv[2])

1 0

Python 3.9.7
by ambv Aug. 30, 2021

Aug. 30, 2021

https://github.com/python/cpython/commit/1016ef37904c7ddc16fb18d3369b2a78cf… commit: 1016ef37904c7ddc16fb18d3369b2a78cf428fa4 branch: 3.9 author: Łukasz Langa <lukasz(a)langa.pl> committer: ambv <lukasz(a)langa.pl> date: 2021-08-30T21:02:15+02:00 summary: Python 3.9.7 files: A Misc/NEWS.d/3.9.7.rst D Misc/NEWS.d/next/Build/2021-06-19-11-50-03.bpo-43298.9ircMb.rst D Misc/NEWS.d/next/Build/2021-06-30-02-32-46.bpo-44535.M9iN4-.rst D Misc/NEWS.d/next/Core and Builtins/2019-12-21-14-18-32.bpo-39091.dOexgQ.rst D Misc/NEWS.d/next/Core and Builtins/2021-05-21-01-42-45.bpo-44184.9qOptC.rst D Misc/NEWS.d/next/Core and Builtins/2021-06-21-11-19-54.bpo-44472.Vvm1yn.rst D Misc/NEWS.d/next/Core and Builtins/2021-06-29-11-49-29.bpo-44523.67-ZIP.rst D Misc/NEWS.d/next/Core and Builtins/2021-07-04-23-38-51.bpo-44562.QdeRQo.rst D Misc/NEWS.d/next/Core and Builtins/2021-07-21-15-26-56.bpo-44698.DA4_0o.rst D Misc/NEWS.d/next/Core and Builtins/2021-08-07-01-26-12.bpo-44856.9rk3li.rst D Misc/NEWS.d/next/Core and Builtins/2021-08-07-21-39-19.bpo-25782.B22lMx.rst D Misc/NEWS.d/next/Core and Builtins/2021-08-09-14-29-52.bpo-33930.--5LQ-.rst D Misc/NEWS.d/next/Core and Builtins/2021-08-09-16-16-03.bpo-44872.OKRlhK.rst D Misc/NEWS.d/next/Core and Builtins/2021-08-11-15-39-57.bpo-44885.i4noUO.rst D Misc/NEWS.d/next/Core and Builtins/2021-08-15-10-39-06.bpo-44698.lITKNc.rst D Misc/NEWS.d/next/Core and Builtins/2021-08-18-19-09-28.bpo-44947.mcvGdS.rst D Misc/NEWS.d/next/Core and Builtins/2021-08-19-14-43-24.bpo-44954.dLn3lg.rst D Misc/NEWS.d/next/Core and Builtins/2021-08-23-19-55-08.bpo-44962.J00ftt.rst D Misc/NEWS.d/next/Core and Builtins/2021-08-26-18-44-03.bpo-45018.pu8H9L.rst D Misc/NEWS.d/next/Documentation/2018-05-19-15-59-29.bpo-33479.4cLlxo.rst D Misc/NEWS.d/next/Documentation/2020-01-30-05-18-48.bpo-39498.Nu3sFL.rst D Misc/NEWS.d/next/Documentation/2021-06-18-06-44-45.bpo-44453.3PIkj2.rst D Misc/NEWS.d/next/Documentation/2021-06-18-18-04-53.bpo-27752.NEByNk.rst D Misc/NEWS.d/next/Documentation/2021-06-24-14-37-16.bpo-43066.Ti7ahX.rst D Misc/NEWS.d/next/Documentation/2021-06-28-12-13-48.bpo-38062.9Ehp9O.rst D Misc/NEWS.d/next/Documentation/2021-07-02-14-02-29.bpo-44544._5_aCz.rst D Misc/NEWS.d/next/Documentation/2021-07-03-18-25-17.bpo-44558.0pTknl.rst D Misc/NEWS.d/next/Documentation/2021-07-15-11-19-03.bpo-42958.gC5IHM.rst D Misc/NEWS.d/next/Documentation/2021-07-18-22-43-14.bpo-44561.T7HpWm.rst D Misc/NEWS.d/next/Documentation/2021-07-20-21-03-18.bpo-30511.eMFkRi.rst D Misc/NEWS.d/next/Documentation/2021-07-22-08-28-03.bpo-35183.p9BWTB.rst D Misc/NEWS.d/next/Documentation/2021-07-25-23-04-15.bpo-44693.JuCbNq.rst D Misc/NEWS.d/next/Documentation/2021-08-06-19-36-21.bpo-44756.1Ngzon.rst D Misc/NEWS.d/next/Documentation/2021-08-13-19-08-03.bpo-44903.aJuvQF.rst D Misc/NEWS.d/next/Library/2017-09-20-14-43-03.bpo-29298._78CSN.rst D Misc/NEWS.d/next/Library/2018-04-24-14-25-07.bpo-33349.Y_0LIr.rst D Misc/NEWS.d/next/Library/2019-06-03-23-53-25.bpo-27513.qITN7d.rst D Misc/NEWS.d/next/Library/2019-09-25-13-54-41.bpo-30256.wBkzox.rst D Misc/NEWS.d/next/Library/2020-01-16-23-41-16.bpo-38840.VzzYZz.rst D Misc/NEWS.d/next/Library/2020-04-24-20-39-38.bpo-34990.3SmL9M.rst D Misc/NEWS.d/next/Library/2020-07-13-23-46-59.bpo-32695.tTqqXe.rst D Misc/NEWS.d/next/Library/2020-07-26-18-17-30.bpo-41402.YRkVkp.rst D Misc/NEWS.d/next/Library/2021-02-06-05-34-01.bpo-43048.yCPUmo.rst D Misc/NEWS.d/next/Library/2021-04-15-12-02-17.bpo-43853.XXCVAp.rst D Misc/NEWS.d/next/Library/2021-05-18-00-17-21.bpo-27334.32EJZi.rst D Misc/NEWS.d/next/Library/2021-06-10-21-53-46.bpo-34266.k3fxnm.rst D Misc/NEWS.d/next/Library/2021-06-12-21-25-35.bpo-27827.TMWh1i.rst D Misc/NEWS.d/next/Library/2021-06-24-19-16-20.bpo-42892.qvRNhI.rst D Misc/NEWS.d/next/Library/2021-06-29-21-17-17.bpo-44461.acqRnV.rst D Misc/NEWS.d/next/Library/2021-07-04-11-33-34.bpo-41249.sHdwBE.rst D Misc/NEWS.d/next/Library/2021-07-04-21-16-53.bpo-44558.cm7Slv.rst D Misc/NEWS.d/next/Library/2021-07-05-18-13-25.bpo-44566.o51Bd1.rst D Misc/NEWS.d/next/Library/2021-07-09-07-14-37.bpo-41928.Q1jMrr.rst D Misc/NEWS.d/next/Library/2021-07-13-09-01-33.bpo-44608.R3IcM1.rst D Misc/NEWS.d/next/Library/2021-07-16-13-40-31.bpo-40897.aveAre.rst D Misc/NEWS.d/next/Library/2021-07-21-10-43-22.bpo-44666.CEThkv.rst D Misc/NEWS.d/next/Library/2021-07-21-23-16-30.bpo-44704.iqHLxQ.rst D Misc/NEWS.d/next/Library/2021-07-24-02-17-59.bpo-44720.shU5Qm.rst D Misc/NEWS.d/next/Library/2021-07-27-22-11-29.bpo-44752._bvbrZ.rst D Misc/NEWS.d/next/Library/2021-07-28-15-50-59.bpo-42853.8SYiF_.rst D Misc/NEWS.d/next/Library/2021-07-30-23-27-30.bpo-44667.tu0Xrv.rst D Misc/NEWS.d/next/Library/2021-08-02-14-37-32.bpo-44806.wOW_Qn.rst D Misc/NEWS.d/next/Library/2021-08-03-15-02-28.bpo-44815.9AmFfy.rst D Misc/NEWS.d/next/Library/2021-08-04-12-29-00.bpo-44822.zePNXA.rst D Misc/NEWS.d/next/Library/2021-08-06-09-43-50.bpo-44605.q4YSBZ.rst D Misc/NEWS.d/next/Library/2021-08-06-13-00-28.bpo-44849.O78F_f.rst D Misc/NEWS.d/next/Library/2021-08-06-19-15-52.bpo-44581.oFDBTB.rst D Misc/NEWS.d/next/Library/2021-08-09-13-17-10.bpo-38956.owWLNv.rst D Misc/NEWS.d/next/Library/2021-08-19-15-03-54.bpo-44955.1mxFQS.rst D Misc/NEWS.d/next/Library/2021-08-20-11-30-52.bpo-44449.1r2-lS.rst D Misc/NEWS.d/next/Library/2021-08-26-16-25-48.bpo-45001.tn_dKp.rst D Misc/NEWS.d/next/Library/2021-08-27-23-40-51.bpo-43913.Uo1Gt5.rst D Misc/NEWS.d/next/Library/2021-08-29-14-49-22.bpo-41620.WJ6PFL.rst D Misc/NEWS.d/next/Security/2021-05-08-11-50-46.bpo-43124.2CTM6M.rst D Misc/NEWS.d/next/Security/2021-06-29-02-45-53.bpo-44394.A220N1.rst D Misc/NEWS.d/next/Security/2021-06-29-23-40-22.bpo-41180.uTWHv_.rst D Misc/NEWS.d/next/Security/2021-08-29-12-39-44.bpo-42278.jvmQz_.rst D Misc/NEWS.d/next/Tests/2019-09-25-18-10-10.bpo-30256.A5i76Q.rst D Misc/NEWS.d/next/Tests/2021-07-22-16-38-39.bpo-44708.SYNaac.rst D Misc/NEWS.d/next/Tests/2021-07-24-20-09-15.bpo-44734.KKsNOV.rst D Misc/NEWS.d/next/Tests/2021-08-06-00-07-15.bpo-40928.aIwb6G.rst D Misc/NEWS.d/next/Tests/2021-08-06-18-36-04.bpo-44852.sUL8YX.rst D Misc/NEWS.d/next/Tests/2021-08-18-18-30-12.bpo-44949.VE5ENv.rst D Misc/NEWS.d/next/Tests/2021-08-26-14-20-44.bpo-45011.mQZdXU.rst D Misc/NEWS.d/next/Tests/2021-08-27-22-37-19.bpo-25130.ig4oJe.rst D Misc/NEWS.d/next/Tools-Demos/2021-07-28-00-51-55.bpo-44756.pvAajB.rst D Misc/NEWS.d/next/Windows/2020-04-13-15-20-28.bpo-40263.1KKEbu.rst D Misc/NEWS.d/next/Windows/2021-07-13-15-32-49.bpo-44572.gXvhDc.rst D Misc/NEWS.d/next/Windows/2021-08-27-23-50-02.bpo-45007.NIBlVG.rst D Misc/NEWS.d/next/macOS/2021-07-20-22-27-01.bpo-44689.mmT_xH.rst D Misc/NEWS.d/next/macOS/2021-08-30-00-04-10.bpo-45007.pixqUB.rst M Include/patchlevel.h M Lib/pydoc_data/topics.py M README.rst diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 999b7fa057e475..af883f2339d812 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -18,12 +18,12 @@ /*--start constants--*/ #define PY_MAJOR_VERSION 3 #define PY_MINOR_VERSION 9 -#define PY_MICRO_VERSION 6 +#define PY_MICRO_VERSION 7 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.9.6+" +#define PY_VERSION "3.9.7" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 8b8544972e528c..bc69434998ad41 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Mon Jun 28 10:13:28 2021 +# Autogenerated by Sphinx on Mon Aug 30 20:40:44 2021 topics = {'assert': 'The "assert" statement\n' '**********************\n' '\n' @@ -1259,6 +1259,10 @@ 'In the latter case, sequence repetition is performed; a negative\n' 'repetition factor yields an empty sequence.\n' '\n' + 'This operation can be customized using the special "__mul__()" ' + 'and\n' + '"__rmul__()" methods.\n' + '\n' 'The "@" (at) operator is intended to be used for matrix\n' 'multiplication. No builtin Python types implement this operator.\n' '\n' @@ -1274,6 +1278,10 @@ 'result. Division by zero raises the "ZeroDivisionError" ' 'exception.\n' '\n' + 'This operation can be customized using the special "__truediv__()" ' + 'and\n' + '"__floordiv__()" methods.\n' + '\n' 'The "%" (modulo) operator yields the remainder from the division ' 'of\n' 'the first argument by the second. The numeric arguments are ' @@ -1305,6 +1313,10 @@ 'string formatting is described in the Python Library Reference,\n' 'section printf-style String Formatting.\n' '\n' + 'The *modulo* operation can be customized using the special ' + '"__mod__()"\n' + 'method.\n' + '\n' 'The floor division operator, the modulo operator, and the ' '"divmod()"\n' 'function are not defined for complex numbers. Instead, convert to ' @@ -1319,9 +1331,16 @@ 'and then added together. In the latter case, the sequences are\n' 'concatenated.\n' '\n' + 'This operation can be customized using the special "__add__()" ' + 'and\n' + '"__radd__()" methods.\n' + '\n' 'The "-" (subtraction) operator yields the difference of its ' 'arguments.\n' - 'The numeric arguments are first converted to a common type.\n', + 'The numeric arguments are first converted to a common type.\n' + '\n' + 'This operation can be customized using the special "__sub__()" ' + 'method.\n', 'bitwise': 'Binary bitwise operations\n' '*************************\n' '\n' @@ -1334,14 +1353,18 @@ '\n' 'The "&" operator yields the bitwise AND of its arguments, which ' 'must\n' - 'be integers.\n' + 'be integers or one of them must be a custom object overriding\n' + '"__and__()" or "__rand__()" special methods.\n' '\n' 'The "^" operator yields the bitwise XOR (exclusive OR) of its\n' - 'arguments, which must be integers.\n' + 'arguments, which must be integers or one of them must be a ' + 'custom\n' + 'object overriding "__xor__()" or "__rxor__()" special methods.\n' '\n' 'The "|" operator yields the bitwise (inclusive) OR of its ' 'arguments,\n' - 'which must be integers.\n', + 'which must be integers or one of them must be a custom object\n' + 'overriding "__or__()" or "__ror__()" special methods.\n', 'bltin-code-objects': 'Code Objects\n' '************\n' '\n' @@ -1787,7 +1810,11 @@ ' comp_operator ::= "<" | ">" | "==" | ">=" | "<=" | "!="\n' ' | "is" ["not"] | ["not"] "in"\n' '\n' - 'Comparisons yield boolean values: "True" or "False".\n' + 'Comparisons yield boolean values: "True" or "False". Custom ' + '*rich\n' + 'comparison methods* may return non-boolean values. In this ' + 'case Python\n' + 'will call "bool()" on such value in boolean contexts.\n' '\n' 'Comparisons can be chained arbitrarily, e.g., "x < y <= z" ' 'is\n' @@ -7472,7 +7499,10 @@ '"ZeroDivisionError".\n' 'Raising a negative number to a fractional power results in a ' '"complex"\n' - 'number. (In earlier versions it raised a "ValueError".)\n', + 'number. (In earlier versions it raised a "ValueError".)\n' + '\n' + 'This operation can be customized using the special "__pow__()" ' + 'method.\n', 'raise': 'The "raise" statement\n' '*********************\n' '\n' @@ -7872,6 +7902,10 @@ 'the\n' 'second argument.\n' '\n' + 'This operation can be customized using the special ' + '"__lshift__()" and\n' + '"__rshift__()" methods.\n' + '\n' 'A right shift by *n* bits is defined as floor division by ' '"pow(2,n)".\n' 'A left shift by *n* bits is defined as multiplication with ' @@ -10092,7 +10126,7 @@ '*start* and\n' ' *end* are interpreted as in slice notation.\n' '\n' - "str.encode(encoding='utf-8', errors='strict')\n" + 'str.encode(encoding="utf-8", errors="strict")\n' '\n' ' Return an encoded version of the string as a bytes ' 'object. Default\n' @@ -10598,7 +10632,7 @@ 'followed by\n' ' the string itself.\n' '\n' - 'str.rsplit(sep=None, maxsplit=- 1)\n' + 'str.rsplit(sep=None, maxsplit=-1)\n' '\n' ' Return a list of the words in the string, using *sep* ' 'as the\n' @@ -10639,7 +10673,7 @@ " >>> 'Monty Python'.removesuffix(' Python')\n" " 'Monty'\n" '\n' - 'str.split(sep=None, maxsplit=- 1)\n' + 'str.split(sep=None, maxsplit=-1)\n' '\n' ' Return a list of the words in the string, using *sep* ' 'as the\n' @@ -11611,7 +11645,7 @@ ' points. All the code points in the range "U+0000 - ' 'U+10FFFF"\n' ' can be represented in a string. Python doesn’t have a ' - '*char*\n' + '"char"\n' ' type; instead, every code point in the string is ' 'represented\n' ' as a string object with length "1". The built-in ' @@ -13419,7 +13453,7 @@ '| | "s[i:i] = ' '[x]") | |\n' '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.pop([i])" | retrieves the item at *i* ' + '| "s.pop()" or "s.pop(i)" | retrieves the item at *i* ' 'and | (2) |\n' '| | also removes it from ' '*s* | |\n' @@ -13882,7 +13916,7 @@ '| | "s[i:i] = ' '[x]") | |\n' '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.pop([i])" | retrieves the item at ' + '| "s.pop()" or "s.pop(i)" | retrieves the item at ' '*i* and | (2) |\n' '| | also removes it from ' '*s* | |\n' @@ -13947,15 +13981,21 @@ ' u_expr ::= power | "-" u_expr | "+" u_expr | "~" u_expr\n' '\n' 'The unary "-" (minus) operator yields the negation of its numeric\n' - 'argument.\n' + 'argument; the operation can be overridden with the "__neg__()" ' + 'special\n' + 'method.\n' '\n' 'The unary "+" (plus) operator yields its numeric argument ' - 'unchanged.\n' + 'unchanged;\n' + 'the operation can be overridden with the "__pos__()" special ' + 'method.\n' '\n' 'The unary "~" (invert) operator yields the bitwise inversion of ' 'its\n' 'integer argument. The bitwise inversion of "x" is defined as\n' - '"-(x+1)". It only applies to integral numbers.\n' + '"-(x+1)". It only applies to integral numbers or to custom ' + 'objects\n' + 'that override the "__invert__()" special method.\n' '\n' 'In all three cases, if the argument does not have the proper type, ' 'a\n' diff --git a/Misc/NEWS.d/3.9.7.rst b/Misc/NEWS.d/3.9.7.rst new file mode 100644 index 00000000000000..445d97ef3c5218 --- /dev/null +++ b/Misc/NEWS.d/3.9.7.rst @@ -0,0 +1,925 @@ +.. bpo: 42278 +.. date: 2021-08-29-12-39-44 +.. nonce: jvmQz_ +.. release date: 2021-08-30 +.. section: Security + +Replaced usage of :func:`tempfile.mktemp` with +:class:`~tempfile.TemporaryDirectory` to avoid a potential race condition. + +.. + +.. bpo: 41180 +.. date: 2021-06-29-23-40-22 +.. nonce: uTWHv_ +.. section: Security + +Add auditing events to the :mod:`marshal` module, and stop raising +``code.__init__`` events for every unmarshalled code object. Directly +instantiated code objects will continue to raise an event, and audit event +handlers should inspect or collect the raw marshal data. This reduces a +significant performance overhead when loading from ``.pyc`` files. + +.. + +.. bpo: 44394 +.. date: 2021-06-29-02-45-53 +.. nonce: A220N1 +.. section: Security + +Update the vendored copy of libexpat to 2.4.1 (from 2.2.8) to get the fix +for the CVE-2013-0340 "Billion Laughs" vulnerability. This copy is most used +on Windows and macOS. + +.. + +.. bpo: 43124 +.. date: 2021-05-08-11-50-46 +.. nonce: 2CTM6M +.. section: Security + +Made the internal ``putcmd`` function in :mod:`smtplib` sanitize input for +presence of ``\r`` and ``\n`` characters to avoid (unlikely) command +injection. + +.. + +.. bpo: 45018 +.. date: 2021-08-26-18-44-03 +.. nonce: pu8H9L +.. section: Core and Builtins + +Fixed pickling of range iterators that iterated for over 2**32 times. + +.. + +.. bpo: 44962 +.. date: 2021-08-23-19-55-08 +.. nonce: J00ftt +.. section: Core and Builtins + +Fix a race in WeakKeyDictionary, WeakValueDictionary and WeakSet when two +threads attempt to commit the last pending removal. This fixes +asyncio.create_task and fixes a data loss in asyncio.run where +shutdown_asyncgens is not run + +.. + +.. bpo: 44954 +.. date: 2021-08-19-14-43-24 +.. nonce: dLn3lg +.. section: Core and Builtins + +Fixed a corner case bug where the result of ``float.fromhex('0x.8p-1074')`` +was rounded the wrong way. + +.. + +.. bpo: 44947 +.. date: 2021-08-18-19-09-28 +.. nonce: mcvGdS +.. section: Core and Builtins + +Refine the syntax error for trailing commas in import statements. Patch by +Pablo Galindo. + +.. + +.. bpo: 44698 +.. date: 2021-08-15-10-39-06 +.. nonce: lITKNc +.. section: Core and Builtins + +Restore behaviour of complex exponentiation with integer-valued exponent of +type :class:`float` or :class:`complex`. + +.. + +.. bpo: 44885 +.. date: 2021-08-11-15-39-57 +.. nonce: i4noUO +.. section: Core and Builtins + +Correct the ast locations of f-strings with format specs and repeated +expressions. Patch by Pablo Galindo + +.. + +.. bpo: 44872 +.. date: 2021-08-09-16-16-03 +.. nonce: OKRlhK +.. section: Core and Builtins + +Use new trashcan macros (Py_TRASHCAN_BEGIN/END) in frameobject.c instead of +the old ones (Py_TRASHCAN_SAFE_BEGIN/END). + +.. + +.. bpo: 33930 +.. date: 2021-08-09-14-29-52 +.. nonce: --5LQ- +.. section: Core and Builtins + +Fix segmentation fault with deep recursion when cleaning method objects. +Patch by Augusto Goulart and Pablo Galindo. + +.. + +.. bpo: 25782 +.. date: 2021-08-07-21-39-19 +.. nonce: B22lMx +.. section: Core and Builtins + +Fix bug where ``PyErr_SetObject`` hangs when the current exception has a +cycle in its context chain. + +.. + +.. bpo: 44856 +.. date: 2021-08-07-01-26-12 +.. nonce: 9rk3li +.. section: Core and Builtins + +Fix reference leaks in the error paths of ``update_bases()`` and +``__build_class__``. Patch by Pablo Galindo. + +.. + +.. bpo: 44698 +.. date: 2021-07-21-15-26-56 +.. nonce: DA4_0o +.. section: Core and Builtins + +Fix undefined behaviour in complex object exponentiation. + +.. + +.. bpo: 44562 +.. date: 2021-07-04-23-38-51 +.. nonce: QdeRQo +.. section: Core and Builtins + +Remove uses of :c:func:`PyObject_GC_Del` in error path when initializing +:class:`types.GenericAlias`. + +.. + +.. bpo: 44523 +.. date: 2021-06-29-11-49-29 +.. nonce: 67-ZIP +.. section: Core and Builtins + +Remove the pass-through for :func:`hash` of :class:`weakref.proxy` objects +to prevent unintended consequences when the original referred object dies +while the proxy is part of a hashable object. Patch by Pablo Galindo. + +.. + +.. bpo: 44472 +.. date: 2021-06-21-11-19-54 +.. nonce: Vvm1yn +.. section: Core and Builtins + +Fix ltrace functionality when exceptions are raised. Patch by Pablo Galindo + +.. + +.. bpo: 44184 +.. date: 2021-05-21-01-42-45 +.. nonce: 9qOptC +.. section: Core and Builtins + +Fix a crash at Python exit when a deallocator function removes the last +strong reference to a heap type. Patch by Victor Stinner. + +.. + +.. bpo: 39091 +.. date: 2019-12-21-14-18-32 +.. nonce: dOexgQ +.. section: Core and Builtins + +Fix crash when using passing a non-exception to a generator's ``throw()`` +method. Patch by Noah Oxer + +.. + +.. bpo: 41620 +.. date: 2021-08-29-14-49-22 +.. nonce: WJ6PFL +.. section: Library + +:meth:`~unittest.TestCase.run` now always return a +:class:`~unittest.TestResult` instance. Previously it returned ``None`` if +the test class or method was decorated with a skipping decorator. + +.. + +.. bpo: 43913 +.. date: 2021-08-27-23-40-51 +.. nonce: Uo1Gt5 +.. section: Library + +Fix bugs in cleaning up classes and modules in :mod:`unittest`: + +* Functions registered with :func:`~unittest.addModuleCleanup` were not called unless the user defines ``tearDownModule()`` in their test module. +* Functions registered with :meth:`~unittest.TestCase.addClassCleanup` were not called if ``tearDownClass`` is set to ``None``. +* Buffering in :class:`~unittest.TestResult` did not work with functions registered with ``addClassCleanup()`` and ``addModuleCleanup()``. +* Errors in functions registered with ``addClassCleanup()`` and ``addModuleCleanup()`` were not handled correctly in buffered and debug modes. +* Errors in ``setUpModule()`` and functions registered with ``addModuleCleanup()`` were reported in wrong order. +* And several lesser bugs. + +.. + +.. bpo: 45001 +.. date: 2021-08-26-16-25-48 +.. nonce: tn_dKp +.. section: Library + +Made email date parsing more robust against malformed input, namely a +whitespace-only ``Date:`` header. Patch by Wouter Bolsterlee. + +.. + +.. bpo: 44449 +.. date: 2021-08-20-11-30-52 +.. nonce: 1r2-lS +.. section: Library + +Fix a crash in the signal handler of the :mod:`faulthandler` module: no +longer modify the reference count of frame objects. Patch by Victor Stinner. + +.. + +.. bpo: 44955 +.. date: 2021-08-19-15-03-54 +.. nonce: 1mxFQS +.. section: Library + +Method :meth:`~unittest.TestResult.stopTestRun` is now always called in pair +with method :meth:`~unittest.TestResult.startTestRun` for +:class:`~unittest.TestResult` objects implicitly created in +:meth:`~unittest.TestCase.run`. Previously it was not called for test +methods and classes decorated with a skipping decorator. + +.. + +.. bpo: 38956 +.. date: 2021-08-09-13-17-10 +.. nonce: owWLNv +.. section: Library + +:class:`argparse.BooleanOptionalAction`'s default value is no longer printed +twice when used with :class:`argparse.ArgumentDefaultsHelpFormatter`. + +.. + +.. bpo: 44581 +.. date: 2021-08-06-19-15-52 +.. nonce: oFDBTB +.. section: Library + +Upgrade bundled pip to 21.2.3 and setuptools to 57.4.0 + +.. + +.. bpo: 44849 +.. date: 2021-08-06-13-00-28 +.. nonce: O78F_f +.. section: Library + +Fix the :func:`os.set_inheritable` function on FreeBSD 14 for file +descriptor opened with the :data:`~os.O_PATH` flag: ignore the +:data:`~errno.EBADF` error on ``ioctl()``, fallback on the ``fcntl()`` +implementation. Patch by Victor Stinner. + +.. + +.. bpo: 44605 +.. date: 2021-08-06-09-43-50 +.. nonce: q4YSBZ +.. section: Library + +The @functools.total_ordering() decorator now works with metaclasses. + +.. + +.. bpo: 44822 +.. date: 2021-08-04-12-29-00 +.. nonce: zePNXA +.. section: Library + +:mod:`sqlite3` user-defined functions and aggregators returning +:class:`strings <str>` with embedded NUL characters are no longer truncated. +Patch by Erlend E. Aasland. + +.. + +.. bpo: 44815 +.. date: 2021-08-03-15-02-28 +.. nonce: 9AmFfy +.. section: Library + +Always show ``loop=`` arg deprecations in :func:`asyncio.gather` and +:func:`asyncio.sleep` + +.. + +.. bpo: 44806 +.. date: 2021-08-02-14-37-32 +.. nonce: wOW_Qn +.. section: Library + +Non-protocol subclasses of :class:`typing.Protocol` ignore now the +``__init__`` method inherited from protocol base classes. + +.. + +.. bpo: 44667 +.. date: 2021-07-30-23-27-30 +.. nonce: tu0Xrv +.. section: Library + +The :func:`tokenize.tokenize` doesn't incorrectly generate a ``NEWLINE`` +token if the source doesn't end with a new line character but the last line +is a comment, as the function is already generating a ``NL`` token. Patch by +Pablo Galindo + +.. + +.. bpo: 42853 +.. date: 2021-07-28-15-50-59 +.. nonce: 8SYiF_ +.. section: Library + +Fix ``http.client.HTTPSConnection`` fails to download >2GiB data. + +.. + +.. bpo: 44752 +.. date: 2021-07-27-22-11-29 +.. nonce: _bvbrZ +.. section: Library + +:mod:`rcompleter` does not call :func:`getattr` on :class:`property` objects +to avoid the side-effect of evaluating the corresponding method. + +.. + +.. bpo: 44720 +.. date: 2021-07-24-02-17-59 +.. nonce: shU5Qm +.. section: Library + +``weakref.proxy`` objects referencing non-iterators now raise ``TypeError`` +rather than dereferencing the null ``tp_iternext`` slot and crashing. + +.. + +.. bpo: 44704 +.. date: 2021-07-21-23-16-30 +.. nonce: iqHLxQ +.. section: Library + +The implementation of ``collections.abc.Set._hash()`` now matches that of +``frozenset.__hash__()``. + +.. + +.. bpo: 44666 +.. date: 2021-07-21-10-43-22 +.. nonce: CEThkv +.. section: Library + +Fixed issue in :func:`compileall.compile_file` when ``sys.stdout`` is +redirected. Patch by Stefan Hölzl. + +.. + +.. bpo: 40897 +.. date: 2021-07-16-13-40-31 +.. nonce: aveAre +.. section: Library + +Give priority to using the current class constructor in +:func:`inspect.signature`. Patch by Weipeng Hong. + +.. + +.. bpo: 44608 +.. date: 2021-07-13-09-01-33 +.. nonce: R3IcM1 +.. section: Library + +Fix memory leak in :func:`_tkinter._flatten` if it is called with a sequence +or set, but not list or tuple. + +.. + +.. bpo: 41928 +.. date: 2021-07-09-07-14-37 +.. nonce: Q1jMrr +.. section: Library + +Update :func:`shutil.copyfile` to raise :exc:`FileNotFoundError` instead of +confusing :exc:`IsADirectoryError` when a path ending with a +:const:`os.path.sep` does not exist; :func:`shutil.copy` and +:func:`shutil.copy2` are also affected. + +.. + +.. bpo: 44566 +.. date: 2021-07-05-18-13-25 +.. nonce: o51Bd1 +.. section: Library + +handle StopIteration subclass raised from @contextlib.contextmanager +generator + +.. + +.. bpo: 44558 +.. date: 2021-07-04-21-16-53 +.. nonce: cm7Slv +.. section: Library + +Make the implementation consistency of :func:`~operator.indexOf` between C +and Python versions. Patch by Dong-hee Na. + +.. + +.. bpo: 41249 +.. date: 2021-07-04-11-33-34 +.. nonce: sHdwBE +.. section: Library + +Fixes ``TypedDict`` to work with ``typing.get_type_hints()`` and postponed +evaluation of annotations across modules. + +.. + +.. bpo: 44461 +.. date: 2021-06-29-21-17-17 +.. nonce: acqRnV +.. section: Library + +Fix bug with :mod:`pdb`'s handling of import error due to a package which +does not have a ``__main__`` module + +.. + +.. bpo: 42892 +.. date: 2021-06-24-19-16-20 +.. nonce: qvRNhI +.. section: Library + +Fixed an exception thrown while parsing a malformed multipart email by +:class:`email.message.EmailMessage`. + +.. + +.. bpo: 27827 +.. date: 2021-06-12-21-25-35 +.. nonce: TMWh1i +.. section: Library + +:meth:`pathlib.PureWindowsPath.is_reserved` now identifies a greater range +of reserved filenames, including those with trailing spaces or colons. + +.. + +.. bpo: 34266 +.. date: 2021-06-10-21-53-46 +.. nonce: k3fxnm +.. section: Library + +Handle exceptions from parsing the arg of :mod:`pdb`'s run/restart command. + +.. + +.. bpo: 27334 +.. date: 2021-05-18-00-17-21 +.. nonce: 32EJZi +.. section: Library + +The :mod:`sqlite3` context manager now performs a rollback (thus releasing +the database lock) if commit failed. Patch by Luca Citi and Erlend E. +Aasland. + +.. + +.. bpo: 43853 +.. date: 2021-04-15-12-02-17 +.. nonce: XXCVAp +.. section: Library + +Improved string handling for :mod:`sqlite3` user-defined functions and +aggregates: + +* It is now possible to pass strings with embedded null characters to UDFs +* Conversion failures now correctly raise :exc:`MemoryError` + +Patch by Erlend E. Aasland. + +.. + +.. bpo: 43048 +.. date: 2021-02-06-05-34-01 +.. nonce: yCPUmo +.. section: Library + +Handle `RecursionError` in :class:`~traceback.TracebackException`'s +constructor, so that long exceptions chains are truncated instead of causing +traceback formatting to fail. + +.. + +.. bpo: 41402 +.. date: 2020-07-26-18-17-30 +.. nonce: YRkVkp +.. section: Library + +Fix :meth:`email.message.EmailMessage.set_content` when called with binary +data and ``7bit`` content transfer encoding. + +.. + +.. bpo: 32695 +.. date: 2020-07-13-23-46-59 +.. nonce: tTqqXe +.. section: Library + +The *compresslevel* and *preset* keyword arguments of :func:`tarfile.open` +are now both documented and tested. + +.. + +.. bpo: 34990 +.. date: 2020-04-24-20-39-38 +.. nonce: 3SmL9M +.. section: Library + +Fixed a Y2k38 bug in the compileall module where it would fail to compile +files with a modification time after the year 2038. + +.. + +.. bpo: 38840 +.. date: 2020-01-16-23-41-16 +.. nonce: VzzYZz +.. section: Library + +Fix ``test___all__`` on platforms lacking a shared memory implementation. + +.. + +.. bpo: 30256 +.. date: 2019-09-25-13-54-41 +.. nonce: wBkzox +.. section: Library + +Pass multiprocessing BaseProxy argument ``manager_owned`` through AutoProxy. + +.. + +.. bpo: 27513 +.. date: 2019-06-03-23-53-25 +.. nonce: qITN7d +.. section: Library + +:func:`email.utils.getaddresses` now accepts :class:`email.header.Header` +objects along with string values. Patch by Zackery Spytz. + +.. + +.. bpo: 33349 +.. date: 2018-04-24-14-25-07 +.. nonce: Y_0LIr +.. section: Library + +lib2to3 now recognizes async generators everywhere. + +.. + +.. bpo: 29298 +.. date: 2017-09-20-14-43-03 +.. nonce: _78CSN +.. section: Library + +Fix ``TypeError`` when required subparsers without ``dest`` do not receive +arguments. Patch by Anthony Sottile. + +.. + +.. bpo: 44903 +.. date: 2021-08-13-19-08-03 +.. nonce: aJuvQF +.. section: Documentation + +Removed the othergui.rst file, any references to it, and the list of GUI +frameworks in the FAQ. In their place I've added links to the Python Wiki +`page on GUI frameworks <https://wiki.python.org/moin/GuiProgramming>`. + +.. + +.. bpo: 44756 +.. date: 2021-08-06-19-36-21 +.. nonce: 1Ngzon +.. section: Documentation + +Reverted automated virtual environment creation on ``make html`` when +building documentation. It turned out to be disruptive for downstream +distributors. + +.. + +.. bpo: 44693 +.. date: 2021-07-25-23-04-15 +.. nonce: JuCbNq +.. section: Documentation + +Update the definition of __future__ in the glossary by replacing the +confusing word "pseudo-module" with a more accurate description. + +.. + +.. bpo: 35183 +.. date: 2021-07-22-08-28-03 +.. nonce: p9BWTB +.. section: Documentation + +Add typical examples to os.path.splitext docs + +.. + +.. bpo: 30511 +.. date: 2021-07-20-21-03-18 +.. nonce: eMFkRi +.. section: Documentation + +Clarify that :func:`shutil.make_archive` is not thread-safe due to reliance +on changing the current working directory. + +.. + +.. bpo: 44561 +.. date: 2021-07-18-22-43-14 +.. nonce: T7HpWm +.. section: Documentation + +Update of three expired hyperlinks in Doc/distributing/index.rst: "Project +structure", "Building and packaging the project", and "Uploading the project +to the Python Packaging Index". + +.. + +.. bpo: 42958 +.. date: 2021-07-15-11-19-03 +.. nonce: gC5IHM +.. section: Documentation + +Updated the docstring and docs of :func:`filecmp.cmp` to be more accurate +and less confusing especially in respect to *shallow* arg. + +.. + +.. bpo: 44558 +.. date: 2021-07-03-18-25-17 +.. nonce: 0pTknl +.. section: Documentation + +Match the docstring and python implementation of :func:`~operator.countOf` +to the behavior of its c implementation. + +.. + +.. bpo: 44544 +.. date: 2021-07-02-14-02-29 +.. nonce: _5_aCz +.. section: Documentation + +List all kwargs for :func:`textwrap.wrap`, :func:`textwrap.fill`, and +:func:`textwrap.shorten`. Now, there are nav links to attributes of +:class:`TextWrap`, which makes navigation much easier while minimizing +duplication in the documentation. + +.. + +.. bpo: 38062 +.. date: 2021-06-28-12-13-48 +.. nonce: 9Ehp9O +.. section: Documentation + +Clarify that atexit uses equality comparisons internally. + +.. + +.. bpo: 43066 +.. date: 2021-06-24-14-37-16 +.. nonce: Ti7ahX +.. section: Documentation + +Added a warning to :mod:`zipfile` docs: filename arg with a leading slash +may cause archive to be un-openable on Windows systems. + +.. + +.. bpo: 27752 +.. date: 2021-06-18-18-04-53 +.. nonce: NEByNk +.. section: Documentation + +Documentation of csv.Dialect is more descriptive. + +.. + +.. bpo: 44453 +.. date: 2021-06-18-06-44-45 +.. nonce: 3PIkj2 +.. section: Documentation + +Fix documentation for the return type of :func:`sysconfig.get_path`. + +.. + +.. bpo: 39498 +.. date: 2020-01-30-05-18-48 +.. nonce: Nu3sFL +.. section: Documentation + +Add a "Security Considerations" index which links to standard library +modules that have explicitly documented security considerations. + +.. + +.. bpo: 33479 +.. date: 2018-05-19-15-59-29 +.. nonce: 4cLlxo +.. section: Documentation + +Remove the unqualified claim that tkinter is threadsafe. It has not been +true for several years and likely never was. An explanation of what is true +may be added later, after more discussion, and possibly after patching +_tkinter.c, + +.. + +.. bpo: 25130 +.. date: 2021-08-27-22-37-19 +.. nonce: ig4oJe +.. section: Tests + +Add calls of :func:`gc.collect` in tests to support PyPy. + +.. + +.. bpo: 45011 +.. date: 2021-08-26-14-20-44 +.. nonce: mQZdXU +.. section: Tests + +Made tests relying on the :mod:`_asyncio` C extension module optional to +allow running on alternative Python implementations. Patch by Serhiy +Storchaka. + +.. + +.. bpo: 44949 +.. date: 2021-08-18-18-30-12 +.. nonce: VE5ENv +.. section: Tests + +Fix auto history tests of test_readline: sometimes, the newline character is +not written at the end, so don't expect it in the output. + +.. + +.. bpo: 44852 +.. date: 2021-08-06-18-36-04 +.. nonce: sUL8YX +.. section: Tests + +Add ability to wholesale silence DeprecationWarnings while running the +regression test suite. + +.. + +.. bpo: 40928 +.. date: 2021-08-06-00-07-15 +.. nonce: aIwb6G +.. section: Tests + +Notify users running test_decimal regression tests on macOS of potential +harmless "malloc can't allocate region" messages spewed by test_decimal. + +.. + +.. bpo: 44734 +.. date: 2021-07-24-20-09-15 +.. nonce: KKsNOV +.. section: Tests + +Fixed floating point precision issue in turtle tests. + +.. + +.. bpo: 44708 +.. date: 2021-07-22-16-38-39 +.. nonce: SYNaac +.. section: Tests + +Regression tests, when run with -w, are now re-running only the affected +test methods instead of re-running the entire test file. + +.. + +.. bpo: 30256 +.. date: 2019-09-25-18-10-10 +.. nonce: A5i76Q +.. section: Tests + +Add test for nested queues when using ``multiprocessing`` shared objects +``AutoProxy[Queue]`` inside ``ListProxy`` and ``DictProxy`` + +.. + +.. bpo: 44535 +.. date: 2021-06-30-02-32-46 +.. nonce: M9iN4- +.. section: Build + +Enable building using a Visual Studio 2022 install on Windows. + +.. + +.. bpo: 43298 +.. date: 2021-06-19-11-50-03 +.. nonce: 9ircMb +.. section: Build + +Improved error message when building without a Windows SDK installed. + +.. + +.. bpo: 45007 +.. date: 2021-08-27-23-50-02 +.. nonce: NIBlVG +.. section: Windows + +Update to OpenSSL 1.1.1l in Windows build + +.. + +.. bpo: 44572 +.. date: 2021-07-13-15-32-49 +.. nonce: gXvhDc +.. section: Windows + +Avoid consuming standard input in the :mod:`platform` module + +.. + +.. bpo: 40263 +.. date: 2020-04-13-15-20-28 +.. nonce: 1KKEbu +.. section: Windows + +This is a follow-on bug from https://bugs.python.org/issue26903. Once that +is applied we run into an off-by-one assertion problem. The assert was not +correct. + +.. + +.. bpo: 45007 +.. date: 2021-08-30-00-04-10 +.. nonce: pixqUB +.. section: macOS + +Update macOS installer builds to use OpenSSL 1.1.1l. + +.. + +.. bpo: 44689 +.. date: 2021-07-20-22-27-01 +.. nonce: mmT_xH +.. section: macOS + +:meth:`ctypes.util.find_library` now works correctly on macOS 11 Big Sur +even if Python is built on an older version of macOS. Previously, when +built on older macOS systems, ``find_library`` was not able to find macOS +system libraries when running on Big Sur due to changes in how system +libraries are stored. + +.. + +.. bpo: 44756 +.. date: 2021-07-28-00-51-55 +.. nonce: pvAajB +.. section: Tools/Demos + +In the Makefile for documentation (:file:`Doc/Makefile`), the ``build`` rule +is dependent on the ``venv`` rule. Therefore, ``html``, ``latex``, and other +build-dependent rules are also now dependent on ``venv``. The ``venv`` rule +only performs an action if ``$(VENVDIR)`` does not exist. +:file:`Doc/README.rst` was updated; most users now only need to type ``make +html``. diff --git a/Misc/NEWS.d/next/Build/2021-06-19-11-50-03.bpo-43298.9ircMb.rst b/Misc/NEWS.d/next/Build/2021-06-19-11-50-03.bpo-43298.9ircMb.rst deleted file mode 100644 index 3bdc24b147a3ef..00000000000000 --- a/Misc/NEWS.d/next/Build/2021-06-19-11-50-03.bpo-43298.9ircMb.rst +++ /dev/null @@ -1 +0,0 @@ -Improved error message when building without a Windows SDK installed. diff --git a/Misc/NEWS.d/next/Build/2021-06-30-02-32-46.bpo-44535.M9iN4-.rst b/Misc/NEWS.d/next/Build/2021-06-30-02-32-46.bpo-44535.M9iN4-.rst deleted file mode 100644 index e06d0d304852f3..00000000000000 --- a/Misc/NEWS.d/next/Build/2021-06-30-02-32-46.bpo-44535.M9iN4-.rst +++ /dev/null @@ -1 +0,0 @@ -Enable building using a Visual Studio 2022 install on Windows. diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-12-21-14-18-32.bpo-39091.dOexgQ.rst b/Misc/NEWS.d/next/Core and Builtins/2019-12-21-14-18-32.bpo-39091.dOexgQ.rst deleted file mode 100644 index c3b4e810d658b2..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2019-12-21-14-18-32.bpo-39091.dOexgQ.rst +++ /dev/null @@ -1 +0,0 @@ -Fix crash when using passing a non-exception to a generator's ``throw()`` method. Patch by Noah Oxer diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-05-21-01-42-45.bpo-44184.9qOptC.rst b/Misc/NEWS.d/next/Core and Builtins/2021-05-21-01-42-45.bpo-44184.9qOptC.rst deleted file mode 100644 index 3aba9a58475c61..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-05-21-01-42-45.bpo-44184.9qOptC.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a crash at Python exit when a deallocator function removes the last strong -reference to a heap type. -Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-06-21-11-19-54.bpo-44472.Vvm1yn.rst b/Misc/NEWS.d/next/Core and Builtins/2021-06-21-11-19-54.bpo-44472.Vvm1yn.rst deleted file mode 100644 index 34fa2a9e8615fa..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-06-21-11-19-54.bpo-44472.Vvm1yn.rst +++ /dev/null @@ -1 +0,0 @@ -Fix ltrace functionality when exceptions are raised. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-06-29-11-49-29.bpo-44523.67-ZIP.rst b/Misc/NEWS.d/next/Core and Builtins/2021-06-29-11-49-29.bpo-44523.67-ZIP.rst deleted file mode 100644 index aa51a7478583b1..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-06-29-11-49-29.bpo-44523.67-ZIP.rst +++ /dev/null @@ -1,3 +0,0 @@ -Remove the pass-through for :func:`hash` of :class:`weakref.proxy` objects -to prevent unintended consequences when the original referred object -dies while the proxy is part of a hashable object. Patch by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-07-04-23-38-51.bpo-44562.QdeRQo.rst b/Misc/NEWS.d/next/Core and Builtins/2021-07-04-23-38-51.bpo-44562.QdeRQo.rst deleted file mode 100644 index 2fc65bcfdeef43..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-07-04-23-38-51.bpo-44562.QdeRQo.rst +++ /dev/null @@ -1,2 +0,0 @@ -Remove uses of :c:func:`PyObject_GC_Del` in error path when initializing -:class:`types.GenericAlias`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-07-21-15-26-56.bpo-44698.DA4_0o.rst b/Misc/NEWS.d/next/Core and Builtins/2021-07-21-15-26-56.bpo-44698.DA4_0o.rst deleted file mode 100644 index ed389630c8ba11..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-07-21-15-26-56.bpo-44698.DA4_0o.rst +++ /dev/null @@ -1 +0,0 @@ -Fix undefined behaviour in complex object exponentiation. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-07-01-26-12.bpo-44856.9rk3li.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-07-01-26-12.bpo-44856.9rk3li.rst deleted file mode 100644 index 1111d01b726fa2..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-07-01-26-12.bpo-44856.9rk3li.rst +++ /dev/null @@ -1 +0,0 @@ -Fix reference leaks in the error paths of ``update_bases()`` and ``__build_class__``. Patch by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-07-21-39-19.bpo-25782.B22lMx.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-07-21-39-19.bpo-25782.B22lMx.rst deleted file mode 100644 index 1c52059f76c8f3..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-07-21-39-19.bpo-25782.B22lMx.rst +++ /dev/null @@ -1 +0,0 @@ -Fix bug where ``PyErr_SetObject`` hangs when the current exception has a cycle in its context chain. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-09-14-29-52.bpo-33930.--5LQ-.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-09-14-29-52.bpo-33930.--5LQ-.rst deleted file mode 100644 index 827dd3f8b65131..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-09-14-29-52.bpo-33930.--5LQ-.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix segmentation fault with deep recursion when cleaning method objects. -Patch by Augusto Goulart and Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-09-16-16-03.bpo-44872.OKRlhK.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-09-16-16-03.bpo-44872.OKRlhK.rst deleted file mode 100644 index 9a0d00018b2a7c..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-09-16-16-03.bpo-44872.OKRlhK.rst +++ /dev/null @@ -1 +0,0 @@ -Use new trashcan macros (Py_TRASHCAN_BEGIN/END) in frameobject.c instead of the old ones (Py_TRASHCAN_SAFE_BEGIN/END). \ No newline at end of file diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-11-15-39-57.bpo-44885.i4noUO.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-11-15-39-57.bpo-44885.i4noUO.rst deleted file mode 100644 index c6abd7363af711..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-11-15-39-57.bpo-44885.i4noUO.rst +++ /dev/null @@ -1,2 +0,0 @@ -Correct the ast locations of f-strings with format specs and repeated -expressions. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-15-10-39-06.bpo-44698.lITKNc.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-15-10-39-06.bpo-44698.lITKNc.rst deleted file mode 100644 index f197253e10ec69..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-15-10-39-06.bpo-44698.lITKNc.rst +++ /dev/null @@ -1,2 +0,0 @@ -Restore behaviour of complex exponentiation with integer-valued exponent of -type :class:`float` or :class:`complex`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-18-19-09-28.bpo-44947.mcvGdS.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-18-19-09-28.bpo-44947.mcvGdS.rst deleted file mode 100644 index d531ba9faf3de5..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-18-19-09-28.bpo-44947.mcvGdS.rst +++ /dev/null @@ -1,2 +0,0 @@ -Refine the syntax error for trailing commas in import statements. Patch by -Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-19-14-43-24.bpo-44954.dLn3lg.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-19-14-43-24.bpo-44954.dLn3lg.rst deleted file mode 100644 index 4cdeb34b8b6116..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-19-14-43-24.bpo-44954.dLn3lg.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed a corner case bug where the result of ``float.fromhex('0x.8p-1074')`` -was rounded the wrong way. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-23-19-55-08.bpo-44962.J00ftt.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-23-19-55-08.bpo-44962.J00ftt.rst deleted file mode 100644 index 6b4b9dfd8bc3c0..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-23-19-55-08.bpo-44962.J00ftt.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a race in WeakKeyDictionary, WeakValueDictionary and WeakSet when two threads attempt to commit the last pending removal. This fixes asyncio.create_task and fixes a data loss in asyncio.run where shutdown_asyncgens is not run \ No newline at end of file diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-26-18-44-03.bpo-45018.pu8H9L.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-26-18-44-03.bpo-45018.pu8H9L.rst deleted file mode 100644 index 5bf13ef06f34c3..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-26-18-44-03.bpo-45018.pu8H9L.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed pickling of range iterators that iterated for over 2**32 times. diff --git a/Misc/NEWS.d/next/Documentation/2018-05-19-15-59-29.bpo-33479.4cLlxo.rst b/Misc/NEWS.d/next/Documentation/2018-05-19-15-59-29.bpo-33479.4cLlxo.rst deleted file mode 100644 index db4973d3923957..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2018-05-19-15-59-29.bpo-33479.4cLlxo.rst +++ /dev/null @@ -1,4 +0,0 @@ -Remove the unqualified claim that tkinter is threadsafe. It has not been -true for several years and likely never was. An explanation of what is true -may be added later, after more discussion, and possibly after patching -_tkinter.c, diff --git a/Misc/NEWS.d/next/Documentation/2020-01-30-05-18-48.bpo-39498.Nu3sFL.rst b/Misc/NEWS.d/next/Documentation/2020-01-30-05-18-48.bpo-39498.Nu3sFL.rst deleted file mode 100644 index a3e899a80a0fc2..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2020-01-30-05-18-48.bpo-39498.Nu3sFL.rst +++ /dev/null @@ -1 +0,0 @@ -Add a "Security Considerations" index which links to standard library modules that have explicitly documented security considerations. diff --git a/Misc/NEWS.d/next/Documentation/2021-06-18-06-44-45.bpo-44453.3PIkj2.rst b/Misc/NEWS.d/next/Documentation/2021-06-18-06-44-45.bpo-44453.3PIkj2.rst deleted file mode 100644 index fd72cf525c32fb..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-06-18-06-44-45.bpo-44453.3PIkj2.rst +++ /dev/null @@ -1 +0,0 @@ -Fix documentation for the return type of :func:`sysconfig.get_path`. diff --git a/Misc/NEWS.d/next/Documentation/2021-06-18-18-04-53.bpo-27752.NEByNk.rst b/Misc/NEWS.d/next/Documentation/2021-06-18-18-04-53.bpo-27752.NEByNk.rst deleted file mode 100644 index ccb7767a6b6936..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-06-18-18-04-53.bpo-27752.NEByNk.rst +++ /dev/null @@ -1 +0,0 @@ -Documentation of csv.Dialect is more descriptive. diff --git a/Misc/NEWS.d/next/Documentation/2021-06-24-14-37-16.bpo-43066.Ti7ahX.rst b/Misc/NEWS.d/next/Documentation/2021-06-24-14-37-16.bpo-43066.Ti7ahX.rst deleted file mode 100644 index 3e38522839e8be..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-06-24-14-37-16.bpo-43066.Ti7ahX.rst +++ /dev/null @@ -1,2 +0,0 @@ -Added a warning to :mod:`zipfile` docs: filename arg with a leading slash may cause archive to -be un-openable on Windows systems. diff --git a/Misc/NEWS.d/next/Documentation/2021-06-28-12-13-48.bpo-38062.9Ehp9O.rst b/Misc/NEWS.d/next/Documentation/2021-06-28-12-13-48.bpo-38062.9Ehp9O.rst deleted file mode 100644 index 1d90096e20bfa4..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-06-28-12-13-48.bpo-38062.9Ehp9O.rst +++ /dev/null @@ -1 +0,0 @@ -Clarify that atexit uses equality comparisons internally. diff --git a/Misc/NEWS.d/next/Documentation/2021-07-02-14-02-29.bpo-44544._5_aCz.rst b/Misc/NEWS.d/next/Documentation/2021-07-02-14-02-29.bpo-44544._5_aCz.rst deleted file mode 100644 index 4bb69977e0c1a4..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-07-02-14-02-29.bpo-44544._5_aCz.rst +++ /dev/null @@ -1,4 +0,0 @@ -List all kwargs for :func:`textwrap.wrap`, :func:`textwrap.fill`, and -:func:`textwrap.shorten`. Now, there are nav links to attributes of -:class:`TextWrap`, which makes navigation much easier while minimizing -duplication in the documentation. diff --git a/Misc/NEWS.d/next/Documentation/2021-07-03-18-25-17.bpo-44558.0pTknl.rst b/Misc/NEWS.d/next/Documentation/2021-07-03-18-25-17.bpo-44558.0pTknl.rst deleted file mode 100644 index a12a49ccd80cc2..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-07-03-18-25-17.bpo-44558.0pTknl.rst +++ /dev/null @@ -1,2 +0,0 @@ -Match the docstring and python implementation of :func:`~operator.countOf` to the behavior -of its c implementation. diff --git a/Misc/NEWS.d/next/Documentation/2021-07-15-11-19-03.bpo-42958.gC5IHM.rst b/Misc/NEWS.d/next/Documentation/2021-07-15-11-19-03.bpo-42958.gC5IHM.rst deleted file mode 100644 index c93b84d095526e..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-07-15-11-19-03.bpo-42958.gC5IHM.rst +++ /dev/null @@ -1,2 +0,0 @@ -Updated the docstring and docs of :func:`filecmp.cmp` to be more accurate -and less confusing especially in respect to *shallow* arg. diff --git a/Misc/NEWS.d/next/Documentation/2021-07-18-22-43-14.bpo-44561.T7HpWm.rst b/Misc/NEWS.d/next/Documentation/2021-07-18-22-43-14.bpo-44561.T7HpWm.rst deleted file mode 100644 index 53238533edabbf..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-07-18-22-43-14.bpo-44561.T7HpWm.rst +++ /dev/null @@ -1,3 +0,0 @@ -Update of three expired hyperlinks in Doc/distributing/index.rst: -"Project structure", "Building and packaging the project", and "Uploading the -project to the Python Packaging Index". diff --git a/Misc/NEWS.d/next/Documentation/2021-07-20-21-03-18.bpo-30511.eMFkRi.rst b/Misc/NEWS.d/next/Documentation/2021-07-20-21-03-18.bpo-30511.eMFkRi.rst deleted file mode 100644 index a358fb9cc2860b..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-07-20-21-03-18.bpo-30511.eMFkRi.rst +++ /dev/null @@ -1,2 +0,0 @@ -Clarify that :func:`shutil.make_archive` is not thread-safe due to -reliance on changing the current working directory. diff --git a/Misc/NEWS.d/next/Documentation/2021-07-22-08-28-03.bpo-35183.p9BWTB.rst b/Misc/NEWS.d/next/Documentation/2021-07-22-08-28-03.bpo-35183.p9BWTB.rst deleted file mode 100644 index 02c5fe82611fbf..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-07-22-08-28-03.bpo-35183.p9BWTB.rst +++ /dev/null @@ -1 +0,0 @@ -Add typical examples to os.path.splitext docs \ No newline at end of file diff --git a/Misc/NEWS.d/next/Documentation/2021-07-25-23-04-15.bpo-44693.JuCbNq.rst b/Misc/NEWS.d/next/Documentation/2021-07-25-23-04-15.bpo-44693.JuCbNq.rst deleted file mode 100644 index 614abb412df8ea..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-07-25-23-04-15.bpo-44693.JuCbNq.rst +++ /dev/null @@ -1,2 +0,0 @@ -Update the definition of __future__ in the glossary by replacing the confusing -word "pseudo-module" with a more accurate description. diff --git a/Misc/NEWS.d/next/Documentation/2021-08-06-19-36-21.bpo-44756.1Ngzon.rst b/Misc/NEWS.d/next/Documentation/2021-08-06-19-36-21.bpo-44756.1Ngzon.rst deleted file mode 100644 index ca2e1b9b53b752..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-08-06-19-36-21.bpo-44756.1Ngzon.rst +++ /dev/null @@ -1,3 +0,0 @@ -Reverted automated virtual environment creation on ``make html`` when -building documentation. It turned out to be disruptive for downstream -distributors. diff --git a/Misc/NEWS.d/next/Documentation/2021-08-13-19-08-03.bpo-44903.aJuvQF.rst b/Misc/NEWS.d/next/Documentation/2021-08-13-19-08-03.bpo-44903.aJuvQF.rst deleted file mode 100644 index e357405085ca06..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-08-13-19-08-03.bpo-44903.aJuvQF.rst +++ /dev/null @@ -1,3 +0,0 @@ -Removed the othergui.rst file, any references to it, and the list of GUI -frameworks in the FAQ. In their place I've added links to the Python Wiki -`page on GUI frameworks <https://wiki.python.org/moin/GuiProgramming>`. diff --git a/Misc/NEWS.d/next/Library/2017-09-20-14-43-03.bpo-29298._78CSN.rst b/Misc/NEWS.d/next/Library/2017-09-20-14-43-03.bpo-29298._78CSN.rst deleted file mode 100644 index e84c6de02cd27e..00000000000000 --- a/Misc/NEWS.d/next/Library/2017-09-20-14-43-03.bpo-29298._78CSN.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix ``TypeError`` when required subparsers without ``dest`` do not receive -arguments. Patch by Anthony Sottile. diff --git a/Misc/NEWS.d/next/Library/2018-04-24-14-25-07.bpo-33349.Y_0LIr.rst b/Misc/NEWS.d/next/Library/2018-04-24-14-25-07.bpo-33349.Y_0LIr.rst deleted file mode 100644 index be68b3ea7c4a2a..00000000000000 --- a/Misc/NEWS.d/next/Library/2018-04-24-14-25-07.bpo-33349.Y_0LIr.rst +++ /dev/null @@ -1 +0,0 @@ -lib2to3 now recognizes async generators everywhere. diff --git a/Misc/NEWS.d/next/Library/2019-06-03-23-53-25.bpo-27513.qITN7d.rst b/Misc/NEWS.d/next/Library/2019-06-03-23-53-25.bpo-27513.qITN7d.rst deleted file mode 100644 index 90d49bb2a993f1..00000000000000 --- a/Misc/NEWS.d/next/Library/2019-06-03-23-53-25.bpo-27513.qITN7d.rst +++ /dev/null @@ -1,3 +0,0 @@ -:func:`email.utils.getaddresses` now accepts -:class:`email.header.Header` objects along with string values. -Patch by Zackery Spytz. diff --git a/Misc/NEWS.d/next/Library/2019-09-25-13-54-41.bpo-30256.wBkzox.rst b/Misc/NEWS.d/next/Library/2019-09-25-13-54-41.bpo-30256.wBkzox.rst deleted file mode 100644 index 698b0e8a61c0d8..00000000000000 --- a/Misc/NEWS.d/next/Library/2019-09-25-13-54-41.bpo-30256.wBkzox.rst +++ /dev/null @@ -1 +0,0 @@ -Pass multiprocessing BaseProxy argument ``manager_owned`` through AutoProxy. diff --git a/Misc/NEWS.d/next/Library/2020-01-16-23-41-16.bpo-38840.VzzYZz.rst b/Misc/NEWS.d/next/Library/2020-01-16-23-41-16.bpo-38840.VzzYZz.rst deleted file mode 100644 index 727f62b52a710b..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-01-16-23-41-16.bpo-38840.VzzYZz.rst +++ /dev/null @@ -1 +0,0 @@ -Fix ``test___all__`` on platforms lacking a shared memory implementation. diff --git a/Misc/NEWS.d/next/Library/2020-04-24-20-39-38.bpo-34990.3SmL9M.rst b/Misc/NEWS.d/next/Library/2020-04-24-20-39-38.bpo-34990.3SmL9M.rst deleted file mode 100644 index d420b5dce1e3b5..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-04-24-20-39-38.bpo-34990.3SmL9M.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed a Y2k38 bug in the compileall module where it would fail to compile -files with a modification time after the year 2038. diff --git a/Misc/NEWS.d/next/Library/2020-07-13-23-46-59.bpo-32695.tTqqXe.rst b/Misc/NEWS.d/next/Library/2020-07-13-23-46-59.bpo-32695.tTqqXe.rst deleted file mode 100644 index c71316ed656a2b..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-07-13-23-46-59.bpo-32695.tTqqXe.rst +++ /dev/null @@ -1,2 +0,0 @@ -The *compresslevel* and *preset* keyword arguments of :func:`tarfile.open` -are now both documented and tested. diff --git a/Misc/NEWS.d/next/Library/2020-07-26-18-17-30.bpo-41402.YRkVkp.rst b/Misc/NEWS.d/next/Library/2020-07-26-18-17-30.bpo-41402.YRkVkp.rst deleted file mode 100644 index 45585a469e7247..00000000000000 --- a/Misc/NEWS.d/next/Library/2020-07-26-18-17-30.bpo-41402.YRkVkp.rst +++ /dev/null @@ -1 +0,0 @@ -Fix :meth:`email.message.EmailMessage.set_content` when called with binary data and ``7bit`` content transfer encoding. diff --git a/Misc/NEWS.d/next/Library/2021-02-06-05-34-01.bpo-43048.yCPUmo.rst b/Misc/NEWS.d/next/Library/2021-02-06-05-34-01.bpo-43048.yCPUmo.rst deleted file mode 100644 index 99f6b2bbe9e30b..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-02-06-05-34-01.bpo-43048.yCPUmo.rst +++ /dev/null @@ -1 +0,0 @@ -Handle `RecursionError` in :class:`~traceback.TracebackException`'s constructor, so that long exceptions chains are truncated instead of causing traceback formatting to fail. diff --git a/Misc/NEWS.d/next/Library/2021-04-15-12-02-17.bpo-43853.XXCVAp.rst b/Misc/NEWS.d/next/Library/2021-04-15-12-02-17.bpo-43853.XXCVAp.rst deleted file mode 100644 index a0164c4665a056..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-04-15-12-02-17.bpo-43853.XXCVAp.rst +++ /dev/null @@ -1,7 +0,0 @@ -Improved string handling for :mod:`sqlite3` user-defined functions and -aggregates: - -* It is now possible to pass strings with embedded null characters to UDFs -* Conversion failures now correctly raise :exc:`MemoryError` - -Patch by Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2021-05-18-00-17-21.bpo-27334.32EJZi.rst b/Misc/NEWS.d/next/Library/2021-05-18-00-17-21.bpo-27334.32EJZi.rst deleted file mode 100644 index dc0cdf33ec5acf..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-18-00-17-21.bpo-27334.32EJZi.rst +++ /dev/null @@ -1,2 +0,0 @@ -The :mod:`sqlite3` context manager now performs a rollback (thus releasing the -database lock) if commit failed. Patch by Luca Citi and Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2021-06-10-21-53-46.bpo-34266.k3fxnm.rst b/Misc/NEWS.d/next/Library/2021-06-10-21-53-46.bpo-34266.k3fxnm.rst deleted file mode 100644 index 22ef84e9626ad6..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-10-21-53-46.bpo-34266.k3fxnm.rst +++ /dev/null @@ -1 +0,0 @@ -Handle exceptions from parsing the arg of :mod:`pdb`'s run/restart command. diff --git a/Misc/NEWS.d/next/Library/2021-06-12-21-25-35.bpo-27827.TMWh1i.rst b/Misc/NEWS.d/next/Library/2021-06-12-21-25-35.bpo-27827.TMWh1i.rst deleted file mode 100644 index 1b8cc04533ed85..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-12-21-25-35.bpo-27827.TMWh1i.rst +++ /dev/null @@ -1,2 +0,0 @@ -:meth:`pathlib.PureWindowsPath.is_reserved` now identifies a greater range of -reserved filenames, including those with trailing spaces or colons. diff --git a/Misc/NEWS.d/next/Library/2021-06-24-19-16-20.bpo-42892.qvRNhI.rst b/Misc/NEWS.d/next/Library/2021-06-24-19-16-20.bpo-42892.qvRNhI.rst deleted file mode 100644 index 3c70b0534ecabf..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-24-19-16-20.bpo-42892.qvRNhI.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed an exception thrown while parsing a malformed multipart email by :class:`email.message.EmailMessage`. diff --git a/Misc/NEWS.d/next/Library/2021-06-29-21-17-17.bpo-44461.acqRnV.rst b/Misc/NEWS.d/next/Library/2021-06-29-21-17-17.bpo-44461.acqRnV.rst deleted file mode 100644 index 02e25e928b9cff..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-29-21-17-17.bpo-44461.acqRnV.rst +++ /dev/null @@ -1 +0,0 @@ -Fix bug with :mod:`pdb`'s handling of import error due to a package which does not have a ``__main__`` module \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2021-07-04-11-33-34.bpo-41249.sHdwBE.rst b/Misc/NEWS.d/next/Library/2021-07-04-11-33-34.bpo-41249.sHdwBE.rst deleted file mode 100644 index 06dae4a6e93565..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-04-11-33-34.bpo-41249.sHdwBE.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixes ``TypedDict`` to work with ``typing.get_type_hints()`` and postponed evaluation of -annotations across modules. diff --git a/Misc/NEWS.d/next/Library/2021-07-04-21-16-53.bpo-44558.cm7Slv.rst b/Misc/NEWS.d/next/Library/2021-07-04-21-16-53.bpo-44558.cm7Slv.rst deleted file mode 100644 index 647a70490d1bac..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-04-21-16-53.bpo-44558.cm7Slv.rst +++ /dev/null @@ -1,2 +0,0 @@ -Make the implementation consistency of :func:`~operator.indexOf` between -C and Python versions. Patch by Dong-hee Na. diff --git a/Misc/NEWS.d/next/Library/2021-07-05-18-13-25.bpo-44566.o51Bd1.rst b/Misc/NEWS.d/next/Library/2021-07-05-18-13-25.bpo-44566.o51Bd1.rst deleted file mode 100644 index 3b00a1b715feef..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-05-18-13-25.bpo-44566.o51Bd1.rst +++ /dev/null @@ -1 +0,0 @@ -handle StopIteration subclass raised from @contextlib.contextmanager generator \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2021-07-09-07-14-37.bpo-41928.Q1jMrr.rst b/Misc/NEWS.d/next/Library/2021-07-09-07-14-37.bpo-41928.Q1jMrr.rst deleted file mode 100644 index e6bd758980b003..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-09-07-14-37.bpo-41928.Q1jMrr.rst +++ /dev/null @@ -1,4 +0,0 @@ -Update :func:`shutil.copyfile` to raise :exc:`FileNotFoundError` instead of -confusing :exc:`IsADirectoryError` when a path ending with a -:const:`os.path.sep` does not exist; :func:`shutil.copy` and -:func:`shutil.copy2` are also affected. diff --git a/Misc/NEWS.d/next/Library/2021-07-13-09-01-33.bpo-44608.R3IcM1.rst b/Misc/NEWS.d/next/Library/2021-07-13-09-01-33.bpo-44608.R3IcM1.rst deleted file mode 100644 index e0cf948f3cba68..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-13-09-01-33.bpo-44608.R3IcM1.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix memory leak in :func:`_tkinter._flatten` if it is called with a sequence -or set, but not list or tuple. diff --git a/Misc/NEWS.d/next/Library/2021-07-16-13-40-31.bpo-40897.aveAre.rst b/Misc/NEWS.d/next/Library/2021-07-16-13-40-31.bpo-40897.aveAre.rst deleted file mode 100644 index 04f1465f0ac67f..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-16-13-40-31.bpo-40897.aveAre.rst +++ /dev/null @@ -1,2 +0,0 @@ -Give priority to using the current class constructor in -:func:`inspect.signature`. Patch by Weipeng Hong. diff --git a/Misc/NEWS.d/next/Library/2021-07-21-10-43-22.bpo-44666.CEThkv.rst b/Misc/NEWS.d/next/Library/2021-07-21-10-43-22.bpo-44666.CEThkv.rst deleted file mode 100644 index ab2ef22d0c4558..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-21-10-43-22.bpo-44666.CEThkv.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed issue in :func:`compileall.compile_file` when ``sys.stdout`` is redirected. -Patch by Stefan Hölzl. diff --git a/Misc/NEWS.d/next/Library/2021-07-21-23-16-30.bpo-44704.iqHLxQ.rst b/Misc/NEWS.d/next/Library/2021-07-21-23-16-30.bpo-44704.iqHLxQ.rst deleted file mode 100644 index 586661876dedba..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-21-23-16-30.bpo-44704.iqHLxQ.rst +++ /dev/null @@ -1 +0,0 @@ -The implementation of ``collections.abc.Set._hash()`` now matches that of ``frozenset.__hash__()``. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2021-07-24-02-17-59.bpo-44720.shU5Qm.rst b/Misc/NEWS.d/next/Library/2021-07-24-02-17-59.bpo-44720.shU5Qm.rst deleted file mode 100644 index 83694f3988ae9a..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-24-02-17-59.bpo-44720.shU5Qm.rst +++ /dev/null @@ -1 +0,0 @@ -``weakref.proxy`` objects referencing non-iterators now raise ``TypeError`` rather than dereferencing the null ``tp_iternext`` slot and crashing. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2021-07-27-22-11-29.bpo-44752._bvbrZ.rst b/Misc/NEWS.d/next/Library/2021-07-27-22-11-29.bpo-44752._bvbrZ.rst deleted file mode 100644 index 0d8a2cd6a5e0db..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-27-22-11-29.bpo-44752._bvbrZ.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`rcompleter` does not call :func:`getattr` on :class:`property` objects -to avoid the side-effect of evaluating the corresponding method. diff --git a/Misc/NEWS.d/next/Library/2021-07-28-15-50-59.bpo-42853.8SYiF_.rst b/Misc/NEWS.d/next/Library/2021-07-28-15-50-59.bpo-42853.8SYiF_.rst deleted file mode 100644 index aaf8af0fdfa993..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-28-15-50-59.bpo-42853.8SYiF_.rst +++ /dev/null @@ -1 +0,0 @@ -Fix ``http.client.HTTPSConnection`` fails to download >2GiB data. diff --git a/Misc/NEWS.d/next/Library/2021-07-30-23-27-30.bpo-44667.tu0Xrv.rst b/Misc/NEWS.d/next/Library/2021-07-30-23-27-30.bpo-44667.tu0Xrv.rst deleted file mode 100644 index 5b7e20e0afdf5e..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-07-30-23-27-30.bpo-44667.tu0Xrv.rst +++ /dev/null @@ -1,4 +0,0 @@ -The :func:`tokenize.tokenize` doesn't incorrectly generate a ``NEWLINE`` -token if the source doesn't end with a new line character but the last line -is a comment, as the function is already generating a ``NL`` token. Patch by -Pablo Galindo diff --git a/Misc/NEWS.d/next/Library/2021-08-02-14-37-32.bpo-44806.wOW_Qn.rst b/Misc/NEWS.d/next/Library/2021-08-02-14-37-32.bpo-44806.wOW_Qn.rst deleted file mode 100644 index 6d818c3fc57982..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-02-14-37-32.bpo-44806.wOW_Qn.rst +++ /dev/null @@ -1,2 +0,0 @@ -Non-protocol subclasses of :class:`typing.Protocol` ignore now the -``__init__`` method inherited from protocol base classes. diff --git a/Misc/NEWS.d/next/Library/2021-08-03-15-02-28.bpo-44815.9AmFfy.rst b/Misc/NEWS.d/next/Library/2021-08-03-15-02-28.bpo-44815.9AmFfy.rst deleted file mode 100644 index 63dfbf3cd37946..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-03-15-02-28.bpo-44815.9AmFfy.rst +++ /dev/null @@ -1,2 +0,0 @@ -Always show ``loop=`` arg deprecations in :func:`asyncio.gather` and -:func:`asyncio.sleep` diff --git a/Misc/NEWS.d/next/Library/2021-08-04-12-29-00.bpo-44822.zePNXA.rst b/Misc/NEWS.d/next/Library/2021-08-04-12-29-00.bpo-44822.zePNXA.rst deleted file mode 100644 index d078142886d2e0..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-04-12-29-00.bpo-44822.zePNXA.rst +++ /dev/null @@ -1,3 +0,0 @@ -:mod:`sqlite3` user-defined functions and aggregators returning -:class:`strings <str>` with embedded NUL characters are no longer -truncated. Patch by Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2021-08-06-09-43-50.bpo-44605.q4YSBZ.rst b/Misc/NEWS.d/next/Library/2021-08-06-09-43-50.bpo-44605.q4YSBZ.rst deleted file mode 100644 index 93783923e15b3d..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-06-09-43-50.bpo-44605.q4YSBZ.rst +++ /dev/null @@ -1 +0,0 @@ -The @functools.total_ordering() decorator now works with metaclasses. diff --git a/Misc/NEWS.d/next/Library/2021-08-06-13-00-28.bpo-44849.O78F_f.rst b/Misc/NEWS.d/next/Library/2021-08-06-13-00-28.bpo-44849.O78F_f.rst deleted file mode 100644 index b1f225485ddef5..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-06-13-00-28.bpo-44849.O78F_f.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix the :func:`os.set_inheritable` function on FreeBSD 14 for file descriptor -opened with the :data:`~os.O_PATH` flag: ignore the :data:`~errno.EBADF` -error on ``ioctl()``, fallback on the ``fcntl()`` implementation. Patch by -Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2021-08-06-19-15-52.bpo-44581.oFDBTB.rst b/Misc/NEWS.d/next/Library/2021-08-06-19-15-52.bpo-44581.oFDBTB.rst deleted file mode 100644 index 99f08065b9d2f0..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-06-19-15-52.bpo-44581.oFDBTB.rst +++ /dev/null @@ -1 +0,0 @@ -Upgrade bundled pip to 21.2.3 and setuptools to 57.4.0 diff --git a/Misc/NEWS.d/next/Library/2021-08-09-13-17-10.bpo-38956.owWLNv.rst b/Misc/NEWS.d/next/Library/2021-08-09-13-17-10.bpo-38956.owWLNv.rst deleted file mode 100644 index 3f57c0ea5d5a38..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-09-13-17-10.bpo-38956.owWLNv.rst +++ /dev/null @@ -1 +0,0 @@ -:class:`argparse.BooleanOptionalAction`'s default value is no longer printed twice when used with :class:`argparse.ArgumentDefaultsHelpFormatter`. diff --git a/Misc/NEWS.d/next/Library/2021-08-19-15-03-54.bpo-44955.1mxFQS.rst b/Misc/NEWS.d/next/Library/2021-08-19-15-03-54.bpo-44955.1mxFQS.rst deleted file mode 100644 index 57d1da533cde0d..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-19-15-03-54.bpo-44955.1mxFQS.rst +++ /dev/null @@ -1,5 +0,0 @@ -Method :meth:`~unittest.TestResult.stopTestRun` is now always called in pair -with method :meth:`~unittest.TestResult.startTestRun` for -:class:`~unittest.TestResult` objects implicitly created in -:meth:`~unittest.TestCase.run`. Previously it was not called for test -methods and classes decorated with a skipping decorator. diff --git a/Misc/NEWS.d/next/Library/2021-08-20-11-30-52.bpo-44449.1r2-lS.rst b/Misc/NEWS.d/next/Library/2021-08-20-11-30-52.bpo-44449.1r2-lS.rst deleted file mode 100644 index 52f01541fca6a7..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-20-11-30-52.bpo-44449.1r2-lS.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a crash in the signal handler of the :mod:`faulthandler` module: no -longer modify the reference count of frame objects. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2021-08-26-16-25-48.bpo-45001.tn_dKp.rst b/Misc/NEWS.d/next/Library/2021-08-26-16-25-48.bpo-45001.tn_dKp.rst deleted file mode 100644 index 55cc409d0da30f..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-26-16-25-48.bpo-45001.tn_dKp.rst +++ /dev/null @@ -1,2 +0,0 @@ -Made email date parsing more robust against malformed input, namely a -whitespace-only ``Date:`` header. Patch by Wouter Bolsterlee. diff --git a/Misc/NEWS.d/next/Library/2021-08-27-23-40-51.bpo-43913.Uo1Gt5.rst b/Misc/NEWS.d/next/Library/2021-08-27-23-40-51.bpo-43913.Uo1Gt5.rst deleted file mode 100644 index cf3d5ee0e456f1..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-27-23-40-51.bpo-43913.Uo1Gt5.rst +++ /dev/null @@ -1,8 +0,0 @@ -Fix bugs in cleaning up classes and modules in :mod:`unittest`: - -* Functions registered with :func:`~unittest.addModuleCleanup` were not called unless the user defines ``tearDownModule()`` in their test module. -* Functions registered with :meth:`~unittest.TestCase.addClassCleanup` were not called if ``tearDownClass`` is set to ``None``. -* Buffering in :class:`~unittest.TestResult` did not work with functions registered with ``addClassCleanup()`` and ``addModuleCleanup()``. -* Errors in functions registered with ``addClassCleanup()`` and ``addModuleCleanup()`` were not handled correctly in buffered and debug modes. -* Errors in ``setUpModule()`` and functions registered with ``addModuleCleanup()`` were reported in wrong order. -* And several lesser bugs. diff --git a/Misc/NEWS.d/next/Library/2021-08-29-14-49-22.bpo-41620.WJ6PFL.rst b/Misc/NEWS.d/next/Library/2021-08-29-14-49-22.bpo-41620.WJ6PFL.rst deleted file mode 100644 index 7674d4c9532a27..00000000000000 --- a/Misc/NEWS.d/next/Library/2021-08-29-14-49-22.bpo-41620.WJ6PFL.rst +++ /dev/null @@ -1,3 +0,0 @@ -:meth:`~unittest.TestCase.run` now always return a -:class:`~unittest.TestResult` instance. Previously it returned ``None`` if -the test class or method was decorated with a skipping decorator. diff --git a/Misc/NEWS.d/next/Security/2021-05-08-11-50-46.bpo-43124.2CTM6M.rst b/Misc/NEWS.d/next/Security/2021-05-08-11-50-46.bpo-43124.2CTM6M.rst deleted file mode 100644 index e897d6cd3641d7..00000000000000 --- a/Misc/NEWS.d/next/Security/2021-05-08-11-50-46.bpo-43124.2CTM6M.rst +++ /dev/null @@ -1,2 +0,0 @@ -Made the internal ``putcmd`` function in :mod:`smtplib` sanitize input for -presence of ``\r`` and ``\n`` characters to avoid (unlikely) command injection. diff --git a/Misc/NEWS.d/next/Security/2021-06-29-02-45-53.bpo-44394.A220N1.rst b/Misc/NEWS.d/next/Security/2021-06-29-02-45-53.bpo-44394.A220N1.rst deleted file mode 100644 index e32563d2535c7e..00000000000000 --- a/Misc/NEWS.d/next/Security/2021-06-29-02-45-53.bpo-44394.A220N1.rst +++ /dev/null @@ -1,3 +0,0 @@ -Update the vendored copy of libexpat to 2.4.1 (from 2.2.8) to get the fix -for the CVE-2013-0340 "Billion Laughs" vulnerability. This copy is most used -on Windows and macOS. diff --git a/Misc/NEWS.d/next/Security/2021-06-29-23-40-22.bpo-41180.uTWHv_.rst b/Misc/NEWS.d/next/Security/2021-06-29-23-40-22.bpo-41180.uTWHv_.rst deleted file mode 100644 index 88b70c7cea2610..00000000000000 --- a/Misc/NEWS.d/next/Security/2021-06-29-23-40-22.bpo-41180.uTWHv_.rst +++ /dev/null @@ -1,5 +0,0 @@ -Add auditing events to the :mod:`marshal` module, and stop raising -``code.__init__`` events for every unmarshalled code object. Directly -instantiated code objects will continue to raise an event, and audit event -handlers should inspect or collect the raw marshal data. This reduces a -significant performance overhead when loading from ``.pyc`` files. diff --git a/Misc/NEWS.d/next/Security/2021-08-29-12-39-44.bpo-42278.jvmQz_.rst b/Misc/NEWS.d/next/Security/2021-08-29-12-39-44.bpo-42278.jvmQz_.rst deleted file mode 100644 index db880cd9026da4..00000000000000 --- a/Misc/NEWS.d/next/Security/2021-08-29-12-39-44.bpo-42278.jvmQz_.rst +++ /dev/null @@ -1,2 +0,0 @@ -Replaced usage of :func:`tempfile.mktemp` with -:class:`~tempfile.TemporaryDirectory` to avoid a potential race condition. diff --git a/Misc/NEWS.d/next/Tests/2019-09-25-18-10-10.bpo-30256.A5i76Q.rst b/Misc/NEWS.d/next/Tests/2019-09-25-18-10-10.bpo-30256.A5i76Q.rst deleted file mode 100644 index 4a7cfd52fcea22..00000000000000 --- a/Misc/NEWS.d/next/Tests/2019-09-25-18-10-10.bpo-30256.A5i76Q.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add test for nested queues when using ``multiprocessing`` shared objects -``AutoProxy[Queue]`` inside ``ListProxy`` and ``DictProxy`` diff --git a/Misc/NEWS.d/next/Tests/2021-07-22-16-38-39.bpo-44708.SYNaac.rst b/Misc/NEWS.d/next/Tests/2021-07-22-16-38-39.bpo-44708.SYNaac.rst deleted file mode 100644 index 8b26c4de64b983..00000000000000 --- a/Misc/NEWS.d/next/Tests/2021-07-22-16-38-39.bpo-44708.SYNaac.rst +++ /dev/null @@ -1,2 +0,0 @@ -Regression tests, when run with -w, are now re-running only the affected -test methods instead of re-running the entire test file. diff --git a/Misc/NEWS.d/next/Tests/2021-07-24-20-09-15.bpo-44734.KKsNOV.rst b/Misc/NEWS.d/next/Tests/2021-07-24-20-09-15.bpo-44734.KKsNOV.rst deleted file mode 100644 index 94e9ce08f4bf6b..00000000000000 --- a/Misc/NEWS.d/next/Tests/2021-07-24-20-09-15.bpo-44734.KKsNOV.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed floating point precision issue in turtle tests. diff --git a/Misc/NEWS.d/next/Tests/2021-08-06-00-07-15.bpo-40928.aIwb6G.rst b/Misc/NEWS.d/next/Tests/2021-08-06-00-07-15.bpo-40928.aIwb6G.rst deleted file mode 100644 index c9a5e1b01e58aa..00000000000000 --- a/Misc/NEWS.d/next/Tests/2021-08-06-00-07-15.bpo-40928.aIwb6G.rst +++ /dev/null @@ -1,2 +0,0 @@ -Notify users running test_decimal regression tests on macOS of potential -harmless "malloc can't allocate region" messages spewed by test_decimal. diff --git a/Misc/NEWS.d/next/Tests/2021-08-06-18-36-04.bpo-44852.sUL8YX.rst b/Misc/NEWS.d/next/Tests/2021-08-06-18-36-04.bpo-44852.sUL8YX.rst deleted file mode 100644 index 41b5c2fb51c5c7..00000000000000 --- a/Misc/NEWS.d/next/Tests/2021-08-06-18-36-04.bpo-44852.sUL8YX.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add ability to wholesale silence DeprecationWarnings while running the -regression test suite. diff --git a/Misc/NEWS.d/next/Tests/2021-08-18-18-30-12.bpo-44949.VE5ENv.rst b/Misc/NEWS.d/next/Tests/2021-08-18-18-30-12.bpo-44949.VE5ENv.rst deleted file mode 100644 index 7fdf1810b165e5..00000000000000 --- a/Misc/NEWS.d/next/Tests/2021-08-18-18-30-12.bpo-44949.VE5ENv.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix auto history tests of test_readline: sometimes, the newline character is -not written at the end, so don't expect it in the output. diff --git a/Misc/NEWS.d/next/Tests/2021-08-26-14-20-44.bpo-45011.mQZdXU.rst b/Misc/NEWS.d/next/Tests/2021-08-26-14-20-44.bpo-45011.mQZdXU.rst deleted file mode 100644 index 64e701e2f29f49..00000000000000 --- a/Misc/NEWS.d/next/Tests/2021-08-26-14-20-44.bpo-45011.mQZdXU.rst +++ /dev/null @@ -1,3 +0,0 @@ -Made tests relying on the :mod:`_asyncio` C extension module optional to -allow running on alternative Python implementations. Patch by Serhiy -Storchaka. diff --git a/Misc/NEWS.d/next/Tests/2021-08-27-22-37-19.bpo-25130.ig4oJe.rst b/Misc/NEWS.d/next/Tests/2021-08-27-22-37-19.bpo-25130.ig4oJe.rst deleted file mode 100644 index 43ce68bef46099..00000000000000 --- a/Misc/NEWS.d/next/Tests/2021-08-27-22-37-19.bpo-25130.ig4oJe.rst +++ /dev/null @@ -1 +0,0 @@ -Add calls of :func:`gc.collect` in tests to support PyPy. diff --git a/Misc/NEWS.d/next/Tools-Demos/2021-07-28-00-51-55.bpo-44756.pvAajB.rst b/Misc/NEWS.d/next/Tools-Demos/2021-07-28-00-51-55.bpo-44756.pvAajB.rst deleted file mode 100644 index 4734f1f5c41691..00000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2021-07-28-00-51-55.bpo-44756.pvAajB.rst +++ /dev/null @@ -1,6 +0,0 @@ -In the Makefile for documentation (:file:`Doc/Makefile`), the ``build`` rule -is dependent on the ``venv`` rule. Therefore, ``html``, ``latex``, and other -build-dependent rules are also now dependent on ``venv``. The ``venv`` rule -only performs an action if ``$(VENVDIR)`` does not exist. -:file:`Doc/README.rst` was updated; most users now only need to type ``make -html``. diff --git a/Misc/NEWS.d/next/Windows/2020-04-13-15-20-28.bpo-40263.1KKEbu.rst b/Misc/NEWS.d/next/Windows/2020-04-13-15-20-28.bpo-40263.1KKEbu.rst deleted file mode 100644 index 0c31606d4928c8..00000000000000 --- a/Misc/NEWS.d/next/Windows/2020-04-13-15-20-28.bpo-40263.1KKEbu.rst +++ /dev/null @@ -1,3 +0,0 @@ -This is a follow-on bug from https://bugs.python.org/issue26903. Once that -is applied we run into an off-by-one assertion problem. The assert was not -correct. diff --git a/Misc/NEWS.d/next/Windows/2021-07-13-15-32-49.bpo-44572.gXvhDc.rst b/Misc/NEWS.d/next/Windows/2021-07-13-15-32-49.bpo-44572.gXvhDc.rst deleted file mode 100644 index 6e074c59b8445b..00000000000000 --- a/Misc/NEWS.d/next/Windows/2021-07-13-15-32-49.bpo-44572.gXvhDc.rst +++ /dev/null @@ -1 +0,0 @@ -Avoid consuming standard input in the :mod:`platform` module \ No newline at end of file diff --git a/Misc/NEWS.d/next/Windows/2021-08-27-23-50-02.bpo-45007.NIBlVG.rst b/Misc/NEWS.d/next/Windows/2021-08-27-23-50-02.bpo-45007.NIBlVG.rst deleted file mode 100644 index fa076ee4c8b8ad..00000000000000 --- a/Misc/NEWS.d/next/Windows/2021-08-27-23-50-02.bpo-45007.NIBlVG.rst +++ /dev/null @@ -1 +0,0 @@ -Update to OpenSSL 1.1.1l in Windows build diff --git a/Misc/NEWS.d/next/macOS/2021-07-20-22-27-01.bpo-44689.mmT_xH.rst b/Misc/NEWS.d/next/macOS/2021-07-20-22-27-01.bpo-44689.mmT_xH.rst deleted file mode 100644 index b1e878d1ee44af..00000000000000 --- a/Misc/NEWS.d/next/macOS/2021-07-20-22-27-01.bpo-44689.mmT_xH.rst +++ /dev/null @@ -1,5 +0,0 @@ - :meth:`ctypes.util.find_library` now works correctly on macOS 11 Big Sur - even if Python is built on an older version of macOS. Previously, when - built on older macOS systems, ``find_library`` was not able to find - macOS system libraries when running on Big Sur due to changes in - how system libraries are stored. diff --git a/Misc/NEWS.d/next/macOS/2021-08-30-00-04-10.bpo-45007.pixqUB.rst b/Misc/NEWS.d/next/macOS/2021-08-30-00-04-10.bpo-45007.pixqUB.rst deleted file mode 100644 index e4f1ac6de313d4..00000000000000 --- a/Misc/NEWS.d/next/macOS/2021-08-30-00-04-10.bpo-45007.pixqUB.rst +++ /dev/null @@ -1 +0,0 @@ -Update macOS installer builds to use OpenSSL 1.1.1l. diff --git a/README.rst b/README.rst index 87dc1937176e4b..a419856dfb4e39 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.9.6 +This is Python version 3.9.7 ============================ .. image:: https://travis-ci.org/python/cpython.svg?branch=3.9

1 0

bpo-44756: Remove misleading NEWS entries of a change that was reverted before release (GH-28075)
by ambv Aug. 30, 2021

Aug. 30, 2021

https://github.com/python/cpython/commit/5246dbc2a12bf8e64e18efee2fdce02a35… commit: 5246dbc2a12bf8e64e18efee2fdce02a350bbf09 branch: main author: Łukasz Langa <lukasz(a)langa.pl> committer: ambv <lukasz(a)langa.pl> date: 2021-08-30T23:54:47+02:00 summary: bpo-44756: Remove misleading NEWS entries of a change that was reverted before release (GH-28075) files: D Misc/NEWS.d/next/Documentation/2021-08-06-19-36-21.bpo-44756.1Ngzon.rst D Misc/NEWS.d/next/Tools-Demos/2021-07-28-00-51-55.bpo-44756.pvAajB.rst diff --git a/Misc/NEWS.d/next/Documentation/2021-08-06-19-36-21.bpo-44756.1Ngzon.rst b/Misc/NEWS.d/next/Documentation/2021-08-06-19-36-21.bpo-44756.1Ngzon.rst deleted file mode 100644 index ca2e1b9b53b752..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2021-08-06-19-36-21.bpo-44756.1Ngzon.rst +++ /dev/null @@ -1,3 +0,0 @@ -Reverted automated virtual environment creation on ``make html`` when -building documentation. It turned out to be disruptive for downstream -distributors. diff --git a/Misc/NEWS.d/next/Tools-Demos/2021-07-28-00-51-55.bpo-44756.pvAajB.rst b/Misc/NEWS.d/next/Tools-Demos/2021-07-28-00-51-55.bpo-44756.pvAajB.rst deleted file mode 100644 index 4734f1f5c41691..00000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2021-07-28-00-51-55.bpo-44756.pvAajB.rst +++ /dev/null @@ -1,6 +0,0 @@ -In the Makefile for documentation (:file:`Doc/Makefile`), the ``build`` rule -is dependent on the ``venv`` rule. Therefore, ``html``, ``latex``, and other -build-dependent rules are also now dependent on ``venv``. The ``venv`` rule -only performs an action if ``$(VENVDIR)`` does not exist. -:file:`Doc/README.rst` was updated; most users now only need to type ``make -html``.

1 0