Mailman 3 August 2017 - Python-checkins

Daily reference leaks (4243df51fe43): sum=5
by solipsis＠pitrou.net 19 Aug '17

19 Aug '17

results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [0, -7, 8] memory blocks, sum=1 test_functools leaked [0, 3, 1] memory blocks, sum=4 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogzHicaG', '--timeout', '7200']

1 0

[3.6] bpo-31232: Backport custom print rshift message (GH-3155)
by Nick Coghlan 18 Aug '17

18 Aug '17

https://github.com/python/cpython/commit/1a05e87ec75436d818f05a5dabcecaea67… commit: 1a05e87ec75436d818f05a5dabcecaea67334cbd branch: 3.6 author: Nick Coghlan <ncoghlan(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-08-19T16:59:38+10:00 summary: [3.6] bpo-31232: Backport custom print rshift message (GH-3155) bpo-30721 added a "Did you mean ...?" suggestion to rshift TypeError messages that triggers when the LHS is a Python C function called "print". Since this custom error message is expected to be triggered primarily by attempts to use Python 2 print redirection syntax in Python 3, and is incredibly unlikely to be encountered otherwise, it is also being backported to the next 3.6 maintenance release. Initial patch by Sanyam Khurana. files: A Misc/NEWS.d/next/Core and Builtins/2017-08-18-15-15-20.bpo-30721.Hmc56z.rst M Lib/test/test_print.py M Objects/abstract.c diff --git a/Lib/test/test_print.py b/Lib/test/test_print.py index 03f13b4edfc..e6434feaf5e 100644 --- a/Lib/test/test_print.py +++ b/Lib/test/test_print.py @@ -1,4 +1,5 @@ import unittest +import sys from io import StringIO from test import support @@ -155,6 +156,38 @@ def test_string_with_excessive_whitespace(self): self.assertIn('print("Hello World", end=" ")', str(context.exception)) + def test_stream_redirection_hint_for_py2_migration(self): + # Test correct hint produced for Py2 redirection syntax + with self.assertRaises(TypeError) as context: + print >> sys.stderr, "message" + self.assertIn('Did you mean "print(<message>, ' + 'file=<output_stream>)"?', str(context.exception)) + + # Test correct hint is produced in the case where RHS implements + # __rrshift__ but returns NotImplemented + with self.assertRaises(TypeError) as context: + print >> 42 + self.assertIn('Did you mean "print(<message>, ' + 'file=<output_stream>)"?', str(context.exception)) + + # Test stream redirection hint is specific to print + with self.assertRaises(TypeError) as context: + max >> sys.stderr + self.assertNotIn('Did you mean ', str(context.exception)) + + # Test stream redirection hint is specific to rshift + with self.assertRaises(TypeError) as context: + print << sys.stderr + self.assertNotIn('Did you mean', str(context.exception)) + + # Ensure right operand implementing rrshift still works + class OverrideRRShift: + def __rrshift__(self, lhs): + return 42 # Force result independent of LHS + + self.assertEqual(print >> OverrideRRShift(), 42) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-08-18-15-15-20.bpo-30721.Hmc56z.rst b/Misc/NEWS.d/next/Core and Builtins/2017-08-18-15-15-20.bpo-30721.Hmc56z.rst new file mode 100644 index 00000000000..da553d654ec --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-08-18-15-15-20.bpo-30721.Hmc56z.rst @@ -0,0 +1,2 @@ +``print`` now shows correct usage hint for using Python 2 redirection +syntax. Patch by Sanyam Khurana. diff --git a/Objects/abstract.c b/Objects/abstract.c index 38de774de3a..6c6c86cd44a 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -809,6 +809,21 @@ binary_op(PyObject *v, PyObject *w, const int op_slot, const char *op_name) PyObject *result = binary_op1(v, w, op_slot); if (result == Py_NotImplemented) { Py_DECREF(result); + + if (op_slot == NB_SLOT(nb_rshift) && + PyCFunction_Check(v) && + strcmp(((PyCFunctionObject *)v)->m_ml->ml_name, "print") == 0) + { + PyErr_Format(PyExc_TypeError, + "unsupported operand type(s) for %.100s: " + "'%.100s' and '%.100s'. Did you mean \"print(<message>, " + "file=<output_stream>)\"?", + op_name, + v->ob_type->tp_name, + w->ob_type->tp_name); + return NULL; + } + return binop_type_error(v, w, op_name); } return result;

1 0

bpo-31234: test_threading: fix ref cycle (#3150) (#3152)
by Victor Stinner 18 Aug '17

18 Aug '17

https://github.com/python/cpython/commit/b50e7683acac36ff16e6c6c2c32d9a15e4… commit: b50e7683acac36ff16e6c6c2c32d9a15e46b5174 branch: 3.6 author: Victor Stinner <victor.stinner(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-08-19T02:32:54+02:00 summary: bpo-31234: test_threading: fix ref cycle (#3150) (#3152) test_bare_raise_in_brand_new_thread() now explicitly breaks a reference cycle to not leak a dangling thread. (cherry picked from commit 3d284c081fc3042036adfe1bf2ce92c34d743b0b) files: M Lib/test/test_threading.py diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 0db028864d2..162a72ea03f 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1066,6 +1066,8 @@ def run(self): thread.join() self.assertIsNotNone(thread.exc) self.assertIsInstance(thread.exc, RuntimeError) + # explicitly break the reference cycle to not leak a dangling thread + thread.exc = None class TimerTests(BaseTestCase):

1 0

bpo-31234: test_threading: fix ref cycle (#3150)
by Victor Stinner 18 Aug '17

18 Aug '17

https://github.com/python/cpython/commit/3d284c081fc3042036adfe1bf2ce92c34d… commit: 3d284c081fc3042036adfe1bf2ce92c34d743b0b branch: master author: Victor Stinner <victor.stinner(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-08-19T01:54:42+02:00 summary: bpo-31234: test_threading: fix ref cycle (#3150) test_bare_raise_in_brand_new_thread() now explicitly breaks a reference cycle to not leak a dangling thread. files: M Lib/test/test_threading.py diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 0f3ac555c2d..800d26f71b2 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1069,6 +1069,8 @@ def run(self): thread.join() self.assertIsNotNone(thread.exc) self.assertIsInstance(thread.exc, RuntimeError) + # explicitly break the reference cycle to not leak a dangling thread + thread.exc = None class TimerTests(BaseTestCase):

1 0

bpo-31235: Fix ResourceWarning in test_logging (#3147) (#3149)
by Victor Stinner 18 Aug '17

18 Aug '17

https://github.com/python/cpython/commit/3e866dfaecaa4eb8f98c12782d2488f681… commit: 3e866dfaecaa4eb8f98c12782d2488f681225c37 branch: 3.6 author: Victor Stinner <victor.stinner(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-08-19T01:11:44+02:00 summary: bpo-31235: Fix ResourceWarning in test_logging (#3147) (#3149) (cherry picked from commit a7719e27b3cad0f2b86cb932a76cbe55c541b02e) files: M Lib/test/test_logging.py diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 474affdaebc..4ec02e10093 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -770,6 +770,7 @@ def stop(self, timeout=None): """ self.close() self._thread.join(timeout) + asyncore.close_all(map=self._map, ignore_all=True) self._thread = None class ControlMixin(object):

1 0

bpo-30947: Update libexpat from 2.2.1 to 2.2.3 (#3106) (#3145)
by Victor Stinner 18 Aug '17

18 Aug '17

https://github.com/python/cpython/commit/ec4ab09b7c0b5070bdb27351f979cbecc4… commit: ec4ab09b7c0b5070bdb27351f979cbecc4636245 branch: 2.7 author: Victor Stinner <victor.stinner(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-08-19T01:06:45+02:00 summary: bpo-30947: Update libexpat from 2.2.1 to 2.2.3 (#3106) (#3145) * bpo-30947: Update libexpat from 2.2.1 to 2.2.3 * Add NEWS entry * Add new loadlibrary.c * expat_external.h: restore include "pyexpatns.h" * PCbuild: add expat/loadlibrary.c * Define XML_POOR_ENTROPY to compile expat Python 2.7 backport: add expat/loadlibrary.c to PC/VS9.0/ project files (_elementtree and pyexpat). (cherry picked from commit 93d0cb58b4da2a88c56f472c6c19491cc7a390df) files: A Misc/NEWS.d/next/Security/2017-08-16-16-35-59.bpo-30947.iNMmm4.rst A Modules/expat/loadlibrary.c M Modules/expat/expat.h M Modules/expat/siphash.h M Modules/expat/xmlparse.c M Modules/expat/xmlrole.c M Modules/expat/xmltok.c M Modules/expat/xmltok_impl.c M PC/VS9.0/_elementtree.vcproj M PC/VS9.0/pyexpat.vcproj M PCbuild/_elementtree.vcxproj M PCbuild/_elementtree.vcxproj.filters M PCbuild/pyexpat.vcxproj M PCbuild/pyexpat.vcxproj.filters M setup.py diff --git a/Misc/NEWS.d/next/Security/2017-08-16-16-35-59.bpo-30947.iNMmm4.rst b/Misc/NEWS.d/next/Security/2017-08-16-16-35-59.bpo-30947.iNMmm4.rst new file mode 100644 index 00000000000..3caca9a79b4 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2017-08-16-16-35-59.bpo-30947.iNMmm4.rst @@ -0,0 +1,2 @@ +Upgrade libexpat embedded copy from version 2.2.1 to 2.2.3 to get security +fixes. diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index 28b0f954d41..7e5bbb7e393 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -24,7 +24,6 @@ extern "C" { struct XML_ParserStruct; typedef struct XML_ParserStruct *XML_Parser; -/* Should this be defined using stdbool.h when C99 is available? */ typedef unsigned char XML_Bool; #define XML_TRUE ((XML_Bool) 1) #define XML_FALSE ((XML_Bool) 0) @@ -1049,7 +1048,7 @@ XML_GetFeatureList(void); */ #define XML_MAJOR_VERSION 2 #define XML_MINOR_VERSION 2 -#define XML_MICRO_VERSION 1 +#define XML_MICRO_VERSION 3 #ifdef __cplusplus } diff --git a/Modules/expat/loadlibrary.c b/Modules/expat/loadlibrary.c new file mode 100644 index 00000000000..ffce868399b --- /dev/null +++ b/Modules/expat/loadlibrary.c @@ -0,0 +1,141 @@ +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 2016 - 2017, Steve Holme, <steve_holme(a)hotmail.com>. + * + * All rights reserved. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF + * THIRD PARTY RIGHTS. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH + * THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * Except as contained in this notice, the name of a copyright holder shall + * not be used in advertising or otherwise to promote the sale, use or other + * dealings in this Software without prior written authorization of the + * copyright holder. + * + ***************************************************************************/ + +#if defined(_WIN32) + +#include <windows.h> +#include <tchar.h> + + +HMODULE _Expat_LoadLibrary(LPCTSTR filename); + + +#if !defined(LOAD_WITH_ALTERED_SEARCH_PATH) +#define LOAD_WITH_ALTERED_SEARCH_PATH 0x00000008 +#endif + +#if !defined(LOAD_LIBRARY_SEARCH_SYSTEM32) +#define LOAD_LIBRARY_SEARCH_SYSTEM32 0x00000800 +#endif + +/* We use our own typedef here since some headers might lack these */ +typedef HMODULE (APIENTRY *LOADLIBRARYEX_FN)(LPCTSTR, HANDLE, DWORD); + +/* See function definitions in winbase.h */ +#ifdef UNICODE +# ifdef _WIN32_WCE +# define LOADLIBARYEX L"LoadLibraryExW" +# else +# define LOADLIBARYEX "LoadLibraryExW" +# endif +#else +# define LOADLIBARYEX "LoadLibraryExA" +#endif + + +/* + * _Expat_LoadLibrary() + * + * This is used to dynamically load DLLs using the most secure method available + * for the version of Windows that we are running on. + * + * Parameters: + * + * filename [in] - The filename or full path of the DLL to load. If only the + * filename is passed then the DLL will be loaded from the + * Windows system directory. + * + * Returns the handle of the module on success; otherwise NULL. + */ +HMODULE _Expat_LoadLibrary(LPCTSTR filename) +{ + HMODULE hModule = NULL; + LOADLIBRARYEX_FN pLoadLibraryEx = NULL; + + /* Get a handle to kernel32 so we can access it's functions at runtime */ + HMODULE hKernel32 = GetModuleHandle(TEXT("kernel32")); + if(!hKernel32) + return NULL; + + /* Attempt to find LoadLibraryEx() which is only available on Windows 2000 + and above */ + pLoadLibraryEx = (LOADLIBRARYEX_FN) GetProcAddress(hKernel32, LOADLIBARYEX); + + /* Detect if there's already a path in the filename and load the library if + there is. Note: Both back slashes and forward slashes have been supported + since the earlier days of DOS at an API level although they are not + supported by command prompt */ + if(_tcspbrk(filename, TEXT("\\/"))) { + /** !checksrc! disable BANNEDFUNC 1 **/ + hModule = pLoadLibraryEx ? + pLoadLibraryEx(filename, NULL, LOAD_WITH_ALTERED_SEARCH_PATH) : + LoadLibrary(filename); + } + /* Detect if KB2533623 is installed, as LOAD_LIBARY_SEARCH_SYSTEM32 is only + supported on Windows Vista, Windows Server 2008, Windows 7 and Windows + Server 2008 R2 with this patch or natively on Windows 8 and above */ + else if(pLoadLibraryEx && GetProcAddress(hKernel32, "AddDllDirectory")) { + /* Load the DLL from the Windows system directory */ + hModule = pLoadLibraryEx(filename, NULL, LOAD_LIBRARY_SEARCH_SYSTEM32); + } + else { + /* Attempt to get the Windows system path */ + UINT systemdirlen = GetSystemDirectory(NULL, 0); + if(systemdirlen) { + /* Allocate space for the full DLL path (Room for the null terminator + is included in systemdirlen) */ + size_t filenamelen = _tcslen(filename); + TCHAR *path = malloc(sizeof(TCHAR) * (systemdirlen + 1 + filenamelen)); + if(path && GetSystemDirectory(path, systemdirlen)) { + /* Calculate the full DLL path */ + _tcscpy(path + _tcslen(path), TEXT("\\")); + _tcscpy(path + _tcslen(path), filename); + + /* Load the DLL from the Windows system directory */ + /** !checksrc! disable BANNEDFUNC 1 **/ + hModule = pLoadLibraryEx ? + pLoadLibraryEx(path, NULL, LOAD_WITH_ALTERED_SEARCH_PATH) : + LoadLibrary(path); + + } + free(path); + } + } + + return hModule; +} + +#else /* defined(_WIN32) */ + +/* ISO C requires a translation unit to contain at least one declaration + [-Wempty-translation-unit] */ +typedef int _TRANSLATION_UNIT_LOAD_LIBRARY_C_NOT_EMTPY; + +#endif /* defined(_WIN32) */ diff --git a/Modules/expat/siphash.h b/Modules/expat/siphash.h index 3c5fef49be8..581872df7b4 100644 --- a/Modules/expat/siphash.h +++ b/Modules/expat/siphash.h @@ -2,9 +2,8 @@ * siphash.h - SipHash-2-4 in a single header file * -------------------------------------------------------------------------- * Derived by William Ahern from the reference implementation[1] published[2] - * by Jean-Philippe Aumasson and Daniel J. Berstein. Licensed in kind. * by Jean-Philippe Aumasson and Daniel J. Berstein. - * Minimal changes by Sebastian Pipping on top, details below. + * Minimal changes by Sebastian Pipping and Victor Stinner on top, see below. * Licensed under the CC0 Public Domain Dedication license. * * 1. https://www.131002.net/siphash/siphash24.c @@ -12,17 +11,25 @@ * -------------------------------------------------------------------------- * HISTORY: * - * 2017-06-18 (Sebastian Pipping) - * - Address lack of stdint.h for Visual Studio 2003 to 2008 + * 2017-07-25 (Vadim Zeitlin) + * - Fix use of SIPHASH_MAIN macro + * + * 2017-07-05 (Sebastian Pipping) + * - Use _SIP_ULL macro to not require a C++11 compiler if compiled as C++ + * - Add const qualifiers at two places + * - Ensure <=80 characters line length (assuming tab width 4) * - * 2017-06-10 (Sebastian Pipping) + * 2017-06-23 (Victor Stinner) + * - Address Win64 compile warnings + * + * 2017-06-18 (Sebastian Pipping) * - Clarify license note in the header * - Address C89 issues: * - Stop using inline keyword (and let compiler decide) - * - Turn integer suffix ULL to UL * - Replace _Bool by int * - Turn macro siphash24 into a function * - Address invalid conversion (void pointer) by explicit cast + * - Address lack of stdint.h for Visual Studio 2003 to 2008 * - Always expose sip24_valid (for self-tests) * * 2012-11-04 - Born. (William Ahern) @@ -90,6 +97,14 @@ #endif +/* + * Workaround to not require a C++11 compiler for using ULL suffix + * if this code is included and compiled as C++; related GCC warning is: + * warning: use of C++11 long long integer constant [-Wlong-long] + */ +#define _SIP_ULL(high, low) (((uint64_t)high << 32) | low) + + #define SIP_ROTL(x, b) (uint64_t)(((x) << (b)) | ( (x) >> (64 - (b)))) #define SIP_U32TO8_LE(p, v) \ @@ -169,11 +184,12 @@ static void sip_round(struct siphash *H, const int rounds) { } /* sip_round() */ -static struct siphash *sip24_init(struct siphash *H, const struct sipkey *key) { - H->v0 = 0x736f6d6570736575UL ^ key->k[0]; - H->v1 = 0x646f72616e646f6dUL ^ key->k[1]; - H->v2 = 0x6c7967656e657261UL ^ key->k[0]; - H->v3 = 0x7465646279746573UL ^ key->k[1]; +static struct siphash *sip24_init(struct siphash *H, + const struct sipkey *key) { + H->v0 = _SIP_ULL(0x736f6d65U, 0x70736575U) ^ key->k[0]; + H->v1 = _SIP_ULL(0x646f7261U, 0x6e646f6dU) ^ key->k[1]; + H->v2 = _SIP_ULL(0x6c796765U, 0x6e657261U) ^ key->k[0]; + H->v3 = _SIP_ULL(0x74656462U, 0x79746573U) ^ key->k[1]; H->p = H->buf; H->c = 0; @@ -184,7 +200,8 @@ static struct siphash *sip24_init(struct siphash *H, const struct sipkey *key) { #define sip_endof(a) (&(a)[sizeof (a) / sizeof *(a)]) -static struct siphash *sip24_update(struct siphash *H, const void *src, size_t len) { +static struct siphash *sip24_update(struct siphash *H, const void *src, + size_t len) { const unsigned char *p = (const unsigned char *)src, *pe = p + len; uint64_t m; @@ -209,7 +226,7 @@ static struct siphash *sip24_update(struct siphash *H, const void *src, size_t l static uint64_t sip24_final(struct siphash *H) { - char left = H->p - H->buf; + const char left = (char)(H->p - H->buf); uint64_t b = (H->c + left) << 56; switch (left) { @@ -233,7 +250,8 @@ static uint64_t sip24_final(struct siphash *H) { } /* sip24_final() */ -static uint64_t siphash24(const void *src, size_t len, const struct sipkey *key) { +static uint64_t siphash24(const void *src, size_t len, + const struct sipkey *key) { struct siphash state = SIPHASH_INITIALIZER; return sip24_final(sip24_update(sip24_init(&state, key), src, len)); } /* siphash24() */ @@ -321,10 +339,11 @@ static int sip24_valid(void) { struct sipkey k; size_t i; - sip_tokey(&k, "\000\001\002\003\004\005\006\007\010\011\012\013\014\015\016\017"); + sip_tokey(&k, "\000\001\002\003\004\005\006\007\010\011" + "\012\013\014\015\016\017"); for (i = 0; i < sizeof in; ++i) { - in[i] = i; + in[i] = (unsigned char)i; if (siphash24(in, i, &k) != SIP_U8TO64_LE(vectors[i])) return 0; @@ -334,12 +353,12 @@ static int sip24_valid(void) { } /* sip24_valid() */ -#if SIPHASH_MAIN +#ifdef SIPHASH_MAIN #include <stdio.h> int main(void) { - int ok = sip24_valid(); + const int ok = sip24_valid(); if (ok) puts("OK"); diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index daec151e232..b703e61a040 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -1,10 +1,12 @@ /* Copyright (c) 1998, 1999, 2000 Thai Open Source Software Center Ltd See the file COPYING for copying permission. - 77fea421d361dca90041d0040ecf1dca651167fadf2af79e990e35168d70d933 (2.2.1+) + 101bfd65d1ff3d1511cf6671e6aae65f82cd97df6f4da137d46d510731830ad9 (2.2.3+) */ -#define _GNU_SOURCE 1 /* syscall prototype */ +#if !defined(_GNU_SOURCE) +# define _GNU_SOURCE 1 /* syscall prototype */ +#endif #include <stddef.h> #include <string.h> /* memset(), memcpy() */ @@ -19,6 +21,8 @@ #include <sys/time.h> /* gettimeofday() */ #include <sys/types.h> /* getpid() */ #include <unistd.h> /* getpid() */ +#include <fcntl.h> /* O_RDONLY */ +#include <errno.h> #endif #define XML_BUILDING_EXPAT 1 @@ -33,6 +37,57 @@ #include "expat.h" #include "siphash.h" +#if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) +# if defined(HAVE_GETRANDOM) +# include <sys/random.h> /* getrandom */ +# else +# include <unistd.h> /* syscall */ +# include <sys/syscall.h> /* SYS_getrandom */ +# endif +# if ! defined(GRND_NONBLOCK) +# define GRND_NONBLOCK 0x0001 +# endif /* defined(GRND_NONBLOCK) */ +#endif /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */ + +#if defined(HAVE_LIBBSD) \ + && (defined(HAVE_ARC4RANDOM_BUF) || defined(HAVE_ARC4RANDOM)) +# include <bsd/stdlib.h> +#endif + +#if defined(_WIN32) && !defined(LOAD_LIBRARY_SEARCH_SYSTEM32) +# define LOAD_LIBRARY_SEARCH_SYSTEM32 0x00000800 +#endif + +#if !defined(HAVE_GETRANDOM) && !defined(HAVE_SYSCALL_GETRANDOM) \ + && !defined(HAVE_ARC4RANDOM_BUF) && !defined(HAVE_ARC4RANDOM) \ + && !defined(XML_DEV_URANDOM) \ + && !defined(_WIN32) \ + && !defined(XML_POOR_ENTROPY) +# error \ + You do not have support for any sources of high quality entropy \ + enabled. For end user security, that is probably not what you want. \ + \ + Your options include: \ + * Linux + glibc >=2.25 (getrandom): HAVE_GETRANDOM, \ + * Linux + glibc <2.25 (syscall SYS_getrandom): HAVE_SYSCALL_GETRANDOM, \ + * BSD / macOS >=10.7 (arc4random_buf): HAVE_ARC4RANDOM_BUF, \ + * BSD / macOS <10.7 (arc4random): HAVE_ARC4RANDOM, \ + * libbsd (arc4random_buf): HAVE_ARC4RANDOM_BUF + HAVE_LIBBSD, \ + * libbsd (arc4random): HAVE_ARC4RANDOM + HAVE_LIBBSD, \ + * Linux / BSD / macOS (/dev/urandom): XML_DEV_URANDOM \ + * Windows (RtlGenRandom): _WIN32. \ + \ + If insist on not using any of these, bypass this error by defining \ + XML_POOR_ENTROPY; you have been warned. \ + \ + For CMake, one way to pass the define is: \ + cmake -DCMAKE_C_FLAGS="-pipe -O2 -DHAVE_SYSCALL_GETRANDOM" . \ + \ + If you have reasons to patch this detection code away or need changes \ + to the build system, please open a bug. Thank you! +#endif + + #ifdef XML_UNICODE #define XML_ENCODE_MAX XML_UTF16_ENCODE_MAX #define XmlConvert XmlUtf16Convert @@ -436,6 +491,9 @@ static ELEMENT_TYPE * getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr, const char *end); +static XML_Char *copyString(const XML_Char *s, + const XML_Memory_Handling_Suite *memsuite); + static unsigned long generate_hash_secret_salt(XML_Parser parser); static XML_Bool startParsing(XML_Parser parser); @@ -696,21 +754,13 @@ static const XML_Char implicitContext[] = { #if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) -# include <errno.h> - -# if defined(HAVE_GETRANDOM) -# include <sys/random.h> /* getrandom */ -# else -# include <unistd.h> /* syscall */ -# include <sys/syscall.h> /* SYS_getrandom */ -# endif /* Obtain entropy on Linux 3.17+ */ static int -writeRandomBytes_getrandom(void * target, size_t count) { +writeRandomBytes_getrandom_nonblock(void * target, size_t count) { int success = 0; /* full count bytes written? */ size_t bytesWrittenTotal = 0; - const unsigned int getrandomFlags = 0; + const unsigned int getrandomFlags = GRND_NONBLOCK; do { void * const currentTarget = (void*)((char*)target + bytesWrittenTotal); @@ -728,7 +778,7 @@ writeRandomBytes_getrandom(void * target, size_t count) { if (bytesWrittenTotal >= count) success = 1; } - } while (! success && (errno == EINTR || errno == EAGAIN)); + } while (! success && (errno == EINTR)); return success; } @@ -736,12 +786,67 @@ writeRandomBytes_getrandom(void * target, size_t count) { #endif /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */ +#if ! defined(_WIN32) && defined(XML_DEV_URANDOM) + +/* Extract entropy from /dev/urandom */ +static int +writeRandomBytes_dev_urandom(void * target, size_t count) { + int success = 0; /* full count bytes written? */ + size_t bytesWrittenTotal = 0; + + const int fd = open("/dev/urandom", O_RDONLY); + if (fd < 0) { + return 0; + } + + do { + void * const currentTarget = (void*)((char*)target + bytesWrittenTotal); + const size_t bytesToWrite = count - bytesWrittenTotal; + + const ssize_t bytesWrittenMore = read(fd, currentTarget, bytesToWrite); + + if (bytesWrittenMore > 0) { + bytesWrittenTotal += bytesWrittenMore; + if (bytesWrittenTotal >= count) + success = 1; + } + } while (! success && (errno == EINTR)); + + close(fd); + return success; +} + +#endif /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */ + + +#if defined(HAVE_ARC4RANDOM) + +static void +writeRandomBytes_arc4random(void * target, size_t count) { + size_t bytesWrittenTotal = 0; + + while (bytesWrittenTotal < count) { + const uint32_t random32 = arc4random(); + size_t i = 0; + + for (; (i < sizeof(random32)) && (bytesWrittenTotal < count); + i++, bytesWrittenTotal++) { + const uint8_t random8 = (uint8_t)(random32 >> (i * 8)); + ((uint8_t *)target)[bytesWrittenTotal] = random8; + } + } +} + +#endif /* defined(HAVE_ARC4RANDOM) */ + + #ifdef _WIN32 typedef BOOLEAN (APIENTRY *RTLGENRANDOM_FUNC)(PVOID, ULONG); +HMODULE _Expat_LoadLibrary(LPCTSTR filename); /* see loadlibrary.c */ /* Obtain entropy on Windows XP / Windows Server 2003 and later. - * Hint on RtlGenRandom and the following article from libsodioum. + * Hint on RtlGenRandom and the following article from libsodium. * * Michael Howard: Cryptographically Secure Random number on Windows without using CryptoAPI * https://blogs.msdn.microsoft.com/michael_howard/2005/01/14/cryptographicall… @@ -749,7 +854,7 @@ typedef BOOLEAN (APIENTRY *RTLGENRANDOM_FUNC)(PVOID, ULONG); static int writeRandomBytes_RtlGenRandom(void * target, size_t count) { int success = 0; /* full count bytes written? */ - const HMODULE advapi32 = LoadLibrary("ADVAPI32.DLL"); + const HMODULE advapi32 = _Expat_LoadLibrary(TEXT("ADVAPI32.DLL")); if (advapi32) { const RTLGENRANDOM_FUNC RtlGenRandom @@ -768,6 +873,8 @@ writeRandomBytes_RtlGenRandom(void * target, size_t count) { #endif /* _WIN32 */ +#if ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) + static unsigned long gather_time_entropy(void) { @@ -780,16 +887,20 @@ gather_time_entropy(void) int gettimeofday_res; gettimeofday_res = gettimeofday(&tv, NULL); + +#if defined(NDEBUG) + (void)gettimeofday_res; +#else assert (gettimeofday_res == 0); +#endif /* defined(NDEBUG) */ /* Microseconds time is <20 bits entropy */ return tv.tv_usec; #endif } -#if defined(HAVE_ARC4RANDOM_BUF) && defined(HAVE_LIBBSD) -# include <bsd/stdlib.h> -#endif +#endif /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */ + static unsigned long ENTROPY_DEBUG(const char * label, unsigned long entropy) { @@ -808,10 +919,12 @@ generate_hash_secret_salt(XML_Parser parser) { unsigned long entropy; (void)parser; -#if defined(HAVE_ARC4RANDOM_BUF) || defined(__CloudABI__) - (void)gather_time_entropy; +#if defined(HAVE_ARC4RANDOM_BUF) arc4random_buf(&entropy, sizeof(entropy)); return ENTROPY_DEBUG("arc4random_buf", entropy); +#elif defined(HAVE_ARC4RANDOM) + writeRandomBytes_arc4random((void *)&entropy, sizeof(entropy)); + return ENTROPY_DEBUG("arc4random", entropy); #else /* Try high quality providers first .. */ #ifdef _WIN32 @@ -819,10 +932,15 @@ generate_hash_secret_salt(XML_Parser parser) return ENTROPY_DEBUG("RtlGenRandom", entropy); } #elif defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) - if (writeRandomBytes_getrandom((void *)&entropy, sizeof(entropy))) { + if (writeRandomBytes_getrandom_nonblock((void *)&entropy, sizeof(entropy))) { return ENTROPY_DEBUG("getrandom", entropy); } #endif +#if ! defined(_WIN32) && defined(XML_DEV_URANDOM) + if (writeRandomBytes_dev_urandom((void *)&entropy, sizeof(entropy))) { + return ENTROPY_DEBUG("/dev/urandom", entropy); + } +#endif /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */ /* .. and self-made low quality for backup: */ /* Process ID is 0 bits entropy if attacker has local access */ @@ -833,7 +951,7 @@ generate_hash_secret_salt(XML_Parser parser) return ENTROPY_DEBUG("fallback(4)", entropy * 2147483647); } else { return ENTROPY_DEBUG("fallback(8)", - entropy * (unsigned long)2305843009213693951); + entropy * (unsigned long)2305843009213693951ULL); } #endif } @@ -962,6 +1080,8 @@ parserCreate(const XML_Char *encodingName, nsAttsVersion = 0; nsAttsPower = 0; + protocolEncodingName = NULL; + poolInit(&tempPool, &(parser->m_mem)); poolInit(&temp2Pool, &(parser->m_mem)); parserInit(parser, encodingName); @@ -988,9 +1108,9 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { processor = prologInitProcessor; XmlPrologStateInit(&prologState); - protocolEncodingName = (encodingName != NULL - ? poolCopyString(&tempPool, encodingName) - : NULL); + if (encodingName != NULL) { + protocolEncodingName = copyString(encodingName, &(parser->m_mem)); + } curBase = NULL; XmlInitEncoding(&initEncoding, &encoding, 0); userData = NULL; @@ -1103,6 +1223,8 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) unknownEncodingRelease(unknownEncodingData); poolClear(&tempPool); poolClear(&temp2Pool); + FREE((void *)protocolEncodingName); + protocolEncodingName = NULL; parserInit(parser, encodingName); dtdReset(_dtd, &parser->m_mem); return XML_TRUE; @@ -1119,10 +1241,16 @@ XML_SetEncoding(XML_Parser parser, const XML_Char *encodingName) */ if (ps_parsing == XML_PARSING || ps_parsing == XML_SUSPENDED) return XML_STATUS_ERROR; + + /* Get rid of any previous encoding name */ + FREE((void *)protocolEncodingName); + if (encodingName == NULL) + /* No new encoding name */ protocolEncodingName = NULL; else { - protocolEncodingName = poolCopyString(&tempPool, encodingName); + /* Copy the new encoding name into allocated memory */ + protocolEncodingName = copyString(encodingName, &(parser->m_mem)); if (!protocolEncodingName) return XML_STATUS_ERROR; } @@ -1357,6 +1485,7 @@ XML_ParserFree(XML_Parser parser) destroyBindings(inheritedBindings, parser); poolDestroy(&tempPool); poolDestroy(&temp2Pool); + FREE((void *)protocolEncodingName); #ifdef XML_DTD /* external parameter entity parsers share the DTD structure parser->m_dtd with the root parser, so we must not destroy it @@ -1748,7 +1877,8 @@ enum XML_Status XMLCALL XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { if ((parser == NULL) || (len < 0) || ((s == NULL) && (len != 0))) { - errorCode = XML_ERROR_INVALID_ARGUMENT; + if (parser != NULL) + parser->m_errorCode = XML_ERROR_INVALID_ARGUMENT; return XML_STATUS_ERROR; } switch (ps_parsing) { @@ -1783,9 +1913,22 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) if (errorCode == XML_ERROR_NONE) { switch (ps_parsing) { case XML_SUSPENDED: + /* It is hard to be certain, but it seems that this case + * cannot occur. This code is cleaning up a previous parse + * with no new data (since len == 0). Changing the parsing + * state requires getting to execute a handler function, and + * there doesn't seem to be an opportunity for that while in + * this circumstance. + * + * Given the uncertainty, we retain the code but exclude it + * from coverage tests. + * + * LCOV_EXCL_START + */ XmlUpdatePosition(encoding, positionPtr, bufferPtr, &position); positionPtr = bufferPtr; return XML_STATUS_SUSPENDED; + /* LCOV_EXCL_STOP */ case XML_INITIALIZED: case XML_PARSING: ps_parsing = XML_FINISHED; @@ -2974,9 +3117,17 @@ doContent(XML_Parser parser, return XML_ERROR_NO_MEMORY; break; default: + /* All of the tokens produced by XmlContentTok() have their own + * explicit cases, so this default is not strictly necessary. + * However it is a useful safety net, so we retain the code and + * simply exclude it from the coverage tests. + * + * LCOV_EXCL_START + */ if (defaultHandler) reportDefault(parser, enc, s, next); break; + /* LCOV_EXCL_STOP */ } *eventPP = s = next; switch (ps_parsing) { @@ -3067,13 +3218,17 @@ storeAtts(XML_Parser parser, const ENCODING *enc, #endif attsSize = n + nDefaultAtts + INIT_ATTS_SIZE; temp = (ATTRIBUTE *)REALLOC((void *)atts, attsSize * sizeof(ATTRIBUTE)); - if (temp == NULL) + if (temp == NULL) { + attsSize = oldAttsSize; return XML_ERROR_NO_MEMORY; + } atts = temp; #ifdef XML_ATTR_INFO temp2 = (XML_AttrInfo *)REALLOC((void *)attInfo, attsSize * sizeof(XML_AttrInfo)); - if (temp2 == NULL) + if (temp2 == NULL) { + attsSize = oldAttsSize; return XML_ERROR_NO_MEMORY; + } attInfo = temp2; #endif if (n > oldAttsSize) @@ -3210,6 +3365,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, int j; /* hash table index */ unsigned long version = nsAttsVersion; int nsAttsSize = (int)1 << nsAttsPower; + unsigned char oldNsAttsPower = nsAttsPower; /* size of hash table must be at least 2 * (# of prefixed attributes) */ if ((nPrefixes << 1) >> nsAttsPower) { /* true for nsAttsPower = 0 */ NS_ATT *temp; @@ -3219,8 +3375,11 @@ storeAtts(XML_Parser parser, const ENCODING *enc, nsAttsPower = 3; nsAttsSize = (int)1 << nsAttsPower; temp = (NS_ATT *)REALLOC(nsAtts, nsAttsSize * sizeof(NS_ATT)); - if (!temp) + if (!temp) { + /* Restore actual size of memory in nsAtts */ + nsAttsPower = oldNsAttsPower; return XML_ERROR_NO_MEMORY; + } nsAtts = temp; version = 0; /* force re-initialization of nsAtts hash table */ } @@ -3247,8 +3406,23 @@ storeAtts(XML_Parser parser, const ENCODING *enc, ((XML_Char *)s)[-1] = 0; /* clear flag */ id = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, s, 0); - if (!id || !id->prefix) - return XML_ERROR_NO_MEMORY; + if (!id || !id->prefix) { + /* This code is walking through the appAtts array, dealing + * with (in this case) a prefixed attribute name. To be in + * the array, the attribute must have already been bound, so + * has to have passed through the hash table lookup once + * already. That implies that an entry for it already + * exists, so the lookup above will return a pointer to + * already allocated memory. There is no opportunaity for + * the allocator to fail, so the condition above cannot be + * fulfilled. + * + * Since it is difficult to be certain that the above + * analysis is complete, we retain the test and merely + * remove the code from coverage tests. + */ + return XML_ERROR_NO_MEMORY; /* LCOV_EXCL_LINE */ + } b = id->prefix->binding; if (!b) return XML_ERROR_UNBOUND_PREFIX; @@ -3625,8 +3799,16 @@ doCdataSection(XML_Parser parser, } return XML_ERROR_UNCLOSED_CDATA_SECTION; default: + /* Every token returned by XmlCdataSectionTok() has its own + * explicit case, so this default case will never be executed. + * We retain it as a safety net and exclude it from the coverage + * statistics. + * + * LCOV_EXCL_START + */ *eventPP = next; return XML_ERROR_UNEXPECTED_STATE; + /* LCOV_EXCL_STOP */ } *eventPP = s = next; @@ -3686,8 +3868,20 @@ doIgnoreSection(XML_Parser parser, eventEndPP = &eventEndPtr; } else { + /* It's not entirely clear, but it seems the following two lines + * of code cannot be executed. The only occasions on which 'enc' + * is not 'parser->m_encoding' are when this function is called + * from the internal entity processing, and IGNORE sections are an + * error in internal entities. + * + * Since it really isn't clear that this is true, we keep the code + * and just remove it from our coverage tests. + * + * LCOV_EXCL_START + */ eventPP = &(openInternalEntities->internalEventPtr); eventEndPP = &(openInternalEntities->internalEventEndPtr); + /* LCOV_EXCL_STOP */ } *eventPP = s; *startPtr = NULL; @@ -3720,8 +3914,16 @@ doIgnoreSection(XML_Parser parser, } return XML_ERROR_SYNTAX; /* XML_ERROR_UNCLOSED_IGNORE_SECTION */ default: + /* All of the tokens that XmlIgnoreSectionTok() returns have + * explicit cases to handle them, so this default case is never + * executed. We keep it as a safety net anyway, and remove it + * from our test coverage statistics. + * + * LCOV_EXCL_START + */ *eventPP = next; return XML_ERROR_UNEXPECTED_STATE; + /* LCOV_EXCL_STOP */ } /* not reached */ } @@ -3734,6 +3936,7 @@ initializeEncoding(XML_Parser parser) const char *s; #ifdef XML_UNICODE char encodingBuf[128]; + /* See comments abount `protoclEncodingName` in parserInit() */ if (!protocolEncodingName) s = NULL; else { @@ -3817,7 +4020,14 @@ processXmlDecl(XML_Parser parser, int isGeneralTextEntity, reportDefault(parser, encoding, s, next); if (protocolEncodingName == NULL) { if (newEncoding) { - if (newEncoding->minBytesPerChar != encoding->minBytesPerChar) { + /* Check that the specified encoding does not conflict with what + * the parser has already deduced. Do we have the same number + * of bytes in the smallest representation of a character? If + * this is UTF-16, is it the same endianness? + */ + if (newEncoding->minBytesPerChar != encoding->minBytesPerChar + || (newEncoding->minBytesPerChar == 2 && + newEncoding != encoding)) { eventPtr = encodingName; return XML_ERROR_INCORRECT_ENCODING; } @@ -3962,15 +4172,14 @@ entityValueInitProcessor(XML_Parser parser, result = processXmlDecl(parser, 0, start, next); if (result != XML_ERROR_NONE) return result; - switch (ps_parsing) { - case XML_SUSPENDED: - *nextPtr = next; - return XML_ERROR_NONE; - case XML_FINISHED: + /* At this point, ps_parsing cannot be XML_SUSPENDED. For that + * to happen, a parameter entity parsing handler must have + * attempted to suspend the parser, which fails and raises an + * error. The parser can be aborted, but can't be suspended. + */ + if (ps_parsing == XML_FINISHED) return XML_ERROR_ABORTED; - default: - *nextPtr = next; - } + *nextPtr = next; /* stop scanning for text declaration - we found one */ processor = entityValueProcessor; return entityValueProcessor(parser, next, end, nextPtr); @@ -4293,8 +4502,14 @@ doProlog(XML_Parser parser, &dtd->paramEntities, externalSubsetName, sizeof(ENTITY)); - if (!entity) - return XML_ERROR_NO_MEMORY; + if (!entity) { + /* The external subset name "#" will have already been + * inserted into the hash table at the start of the + * external entity parsing, so no allocation will happen + * and lookup() cannot fail. + */ + return XML_ERROR_NO_MEMORY; /* LCOV_EXCL_LINE */ + } if (useForeignDTD) entity->base = curBase; dtd->paramEntityRead = XML_FALSE; @@ -4773,8 +4988,10 @@ doProlog(XML_Parser parser, if (prologState.level >= groupSize) { if (groupSize) { char *temp = (char *)REALLOC(groupConnector, groupSize *= 2); - if (temp == NULL) + if (temp == NULL) { + groupSize /= 2; return XML_ERROR_NO_MEMORY; + } groupConnector = temp; if (dtd->scaffIndex) { int *temp = (int *)REALLOC(dtd->scaffIndex, @@ -4786,8 +5003,10 @@ doProlog(XML_Parser parser, } else { groupConnector = (char *)MALLOC(groupSize = 32); - if (!groupConnector) + if (!groupConnector) { + groupSize = 0; return XML_ERROR_NO_MEMORY; + } } } groupConnector[prologState.level] = 0; @@ -4850,8 +5069,29 @@ doProlog(XML_Parser parser, : !dtd->hasParamEntityRefs)) { if (!entity) return XML_ERROR_UNDEFINED_ENTITY; - else if (!entity->is_internal) - return XML_ERROR_ENTITY_DECLARED_IN_PE; + else if (!entity->is_internal) { + /* It's hard to exhaustively search the code to be sure, + * but there doesn't seem to be a way of executing the + * following line. There are two cases: + * + * If 'standalone' is false, the DTD must have no + * parameter entities or we wouldn't have passed the outer + * 'if' statement. That measn the only entity in the hash + * table is the external subset name "#" which cannot be + * given as a parameter entity name in XML syntax, so the + * lookup must have returned NULL and we don't even reach + * the test for an internal entity. + * + * If 'standalone' is true, it does not seem to be + * possible to create entities taking this code path that + * are not internal entities, so fail the test above. + * + * Because this analysis is very uncertain, the code is + * being left in place and merely removed from the + * coverage test statistics. + */ + return XML_ERROR_ENTITY_DECLARED_IN_PE; /* LCOV_EXCL_LINE */ + } } else if (!entity) { dtd->keepProcessing = dtd->standalone; @@ -5323,11 +5563,15 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, && (poolLength(pool) == 0 || poolLastChar(pool) == 0x20)) break; n = XmlEncode(n, (ICHAR *)buf); - if (!n) { - if (enc == encoding) - eventPtr = ptr; - return XML_ERROR_BAD_CHAR_REF; - } + /* The XmlEncode() functions can never return 0 here. That + * error return happens if the code point passed in is either + * negative or greater than or equal to 0x110000. The + * XmlCharRefNumber() functions will all return a number + * strictly less than 0x110000 or a negative value if an error + * occurred. The negative value is intercepted above, so + * XmlEncode() is never passed a value it might return an + * error for. + */ for (i = 0; i < n; i++) { if (!poolAppendChar(pool, buf[i])) return XML_ERROR_NO_MEMORY; @@ -5401,8 +5645,26 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, break; } if (entity->open) { - if (enc == encoding) - eventPtr = ptr; + if (enc == encoding) { + /* It does not appear that this line can be executed. + * + * The "if (entity->open)" check catches recursive entity + * definitions. In order to be called with an open + * entity, it must have gone through this code before and + * been through the recursive call to + * appendAttributeValue() some lines below. That call + * sets the local encoding ("enc") to the parser's + * internal encoding (internal_utf8 or internal_utf16), + * which can never be the same as the principle encoding. + * It doesn't appear there is another code path that gets + * here with entity->open being TRUE. + * + * Since it is not certain that this logic is watertight, + * we keep the line and merely exclude it from coverage + * tests. + */ + eventPtr = ptr; /* LCOV_EXCL_LINE */ + } return XML_ERROR_RECURSIVE_ENTITY_REF; } if (entity->notation) { @@ -5429,9 +5691,21 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, } break; default: + /* The only token returned by XmlAttributeValueTok() that does + * not have an explicit case here is XML_TOK_PARTIAL_CHAR. + * Getting that would require an entity name to contain an + * incomplete XML character (e.g. \xE2\x82); however previous + * tokenisers will have already recognised and rejected such + * names before XmlAttributeValueTok() gets a look-in. This + * default case should be retained as a safety net, but the code + * excluded from coverage tests. + * + * LCOV_EXCL_START + */ if (enc == encoding) eventPtr = ptr; return XML_ERROR_UNEXPECTED_STATE; + /* LCOV_EXCL_STOP */ } ptr = next; } @@ -5564,12 +5838,15 @@ storeEntityValue(XML_Parser parser, goto endEntityValue; } n = XmlEncode(n, (ICHAR *)buf); - if (!n) { - if (enc == encoding) - eventPtr = entityTextPtr; - result = XML_ERROR_BAD_CHAR_REF; - goto endEntityValue; - } + /* The XmlEncode() functions can never return 0 here. That + * error return happens if the code point passed in is either + * negative or greater than or equal to 0x110000. The + * XmlCharRefNumber() functions will all return a number + * strictly less than 0x110000 or a negative value if an error + * occurred. The negative value is intercepted above, so + * XmlEncode() is never passed a value it might return an + * error for. + */ for (i = 0; i < n; i++) { if (pool->end == pool->ptr && !poolGrow(pool)) { result = XML_ERROR_NO_MEMORY; @@ -5590,10 +5867,18 @@ storeEntityValue(XML_Parser parser, result = XML_ERROR_INVALID_TOKEN; goto endEntityValue; default: + /* This default case should be unnecessary -- all the tokens + * that XmlEntityValueTok() can return have their own explicit + * cases -- but should be retained for safety. We do however + * exclude it from the coverage statistics. + * + * LCOV_EXCL_START + */ if (enc == encoding) eventPtr = entityTextPtr; result = XML_ERROR_UNEXPECTED_STATE; goto endEntityValue; + /* LCOV_EXCL_STOP */ } entityTextPtr = next; } @@ -5691,8 +5976,25 @@ reportDefault(XML_Parser parser, const ENCODING *enc, eventEndPP = &eventEndPtr; } else { + /* To get here, two things must be true; the parser must be + * using a character encoding that is not the same as the + * encoding passed in, and the encoding passed in must need + * conversion to the internal format (UTF-8 unless XML_UNICODE + * is defined). The only occasions on which the encoding passed + * in is not the same as the parser's encoding are when it is + * the internal encoding (e.g. a previously defined parameter + * entity, already converted to internal format). This by + * definition doesn't need conversion, so the whole branch never + * gets executed. + * + * For safety's sake we don't delete these lines and merely + * exclude them from coverage statistics. + * + * LCOV_EXCL_START + */ eventPP = &(openInternalEntities->internalEventPtr); eventEndPP = &(openInternalEntities->internalEventEndPtr); + /* LCOV_EXCL_STOP */ } do { ICHAR *dataPtr = (ICHAR *)dataBuf; @@ -5861,9 +6163,30 @@ getContext(XML_Parser parser) len = dtd->defaultPrefix.binding->uriLen; if (namespaceSeparator) len--; - for (i = 0; i < len; i++) - if (!poolAppendChar(&tempPool, dtd->defaultPrefix.binding->uri[i])) - return NULL; + for (i = 0; i < len; i++) { + if (!poolAppendChar(&tempPool, dtd->defaultPrefix.binding->uri[i])) { + /* Because of memory caching, I don't believe this line can be + * executed. + * + * This is part of a loop copying the default prefix binding + * URI into the parser's temporary string pool. Previously, + * that URI was copied into the same string pool, with a + * terminating NUL character, as part of setContext(). When + * the pool was cleared, that leaves a block definitely big + * enough to hold the URI on the free block list of the pool. + * The URI copy in getContext() therefore cannot run out of + * memory. + * + * If the pool is used between the setContext() and + * getContext() calls, the worst it can do is leave a bigger + * block on the front of the free list. Given that this is + * all somewhat inobvious and program logic can be changed, we + * don't delete the line but we do exclude it from the test + * coverage statistics. + */ + return NULL; /* LCOV_EXCL_LINE */ + } + } needSep = XML_TRUE; } @@ -5875,8 +6198,15 @@ getContext(XML_Parser parser) PREFIX *prefix = (PREFIX *)hashTableIterNext(&iter); if (!prefix) break; - if (!prefix->binding) - continue; + if (!prefix->binding) { + /* This test appears to be (justifiable) paranoia. There does + * not seem to be a way of injecting a prefix without a binding + * that doesn't get errored long before this function is called. + * The test should remain for safety's sake, so we instead + * exclude the following line from the coverage statistics. + */ + continue; /* LCOV_EXCL_LINE */ + } if (needSep && !poolAppendChar(&tempPool, CONTEXT_SEP)) return NULL; for (s = prefix->name; *s; s++) @@ -6547,8 +6877,20 @@ poolCopyString(STRING_POOL *pool, const XML_Char *s) static const XML_Char * poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n) { - if (!pool->ptr && !poolGrow(pool)) - return NULL; + if (!pool->ptr && !poolGrow(pool)) { + /* The following line is unreachable given the current usage of + * poolCopyStringN(). Currently it is called from exactly one + * place to copy the text of a simple general entity. By that + * point, the name of the entity is already stored in the pool, so + * pool->ptr cannot be NULL. + * + * If poolCopyStringN() is used elsewhere as it well might be, + * this line may well become executable again. Regardless, this + * sort of check shouldn't be removed lightly, so we just exclude + * it from the coverage statistics. + */ + return NULL; /* LCOV_EXCL_LINE */ + } for (; n > 0; --n, s++) { if (!poolAppendChar(pool, *s)) return NULL; @@ -6641,8 +6983,19 @@ poolGrow(STRING_POOL *pool) int blockSize = (int)((unsigned)(pool->end - pool->start)*2U); size_t bytesToAllocate; - if (blockSize < 0) - return XML_FALSE; + // NOTE: Needs to be calculated prior to calling `realloc` + // to avoid dangling pointers: + const ptrdiff_t offsetInsideBlock = pool->ptr - pool->start; + + if (blockSize < 0) { + /* This condition traps a situation where either more than + * INT_MAX/2 bytes have already been allocated. This isn't + * readily testable, since it is unlikely that an average + * machine will have that much memory, so we exclude it from the + * coverage statistics. + */ + return XML_FALSE; /* LCOV_EXCL_LINE */ + } bytesToAllocate = poolBytesToAllocateFor(blockSize); if (bytesToAllocate == 0) @@ -6654,7 +7007,7 @@ poolGrow(STRING_POOL *pool) return XML_FALSE; pool->blocks = temp; pool->blocks->size = blockSize; - pool->ptr = pool->blocks->s + (pool->ptr - pool->start); + pool->ptr = pool->blocks->s + offsetInsideBlock; pool->start = pool->blocks->s; pool->end = pool->start + blockSize; } @@ -6663,8 +7016,18 @@ poolGrow(STRING_POOL *pool) int blockSize = (int)(pool->end - pool->start); size_t bytesToAllocate; - if (blockSize < 0) - return XML_FALSE; + if (blockSize < 0) { + /* This condition traps a situation where either more than + * INT_MAX bytes have already been allocated (which is prevented + * by various pieces of program logic, not least this one, never + * mind the unlikelihood of actually having that much memory) or + * the pool control fields have been corrupted (which could + * conceivably happen in an extremely buggy user handler + * function). Either way it isn't readily testable, so we + * exclude it from the coverage statistics. + */ + return XML_FALSE; /* LCOV_EXCL_LINE */ + } if (blockSize < INIT_BLOCK_SIZE) blockSize = INIT_BLOCK_SIZE; @@ -6827,3 +7190,26 @@ getElementType(XML_Parser parser, } return ret; } + +static XML_Char * +copyString(const XML_Char *s, + const XML_Memory_Handling_Suite *memsuite) +{ + int charsRequired = 0; + XML_Char *result; + + /* First determine how long the string is */ + while (s[charsRequired] != 0) { + charsRequired++; + } + /* Include the terminator */ + charsRequired++; + + /* Now allocate space for the copy */ + result = memsuite->malloc_fcn(charsRequired * sizeof(XML_Char)); + if (result == NULL) + return NULL; + /* Copy the original into place */ + memcpy(result, s, charsRequired * sizeof(XML_Char)); + return result; +} diff --git a/Modules/expat/xmlrole.c b/Modules/expat/xmlrole.c index a7c56302796..c809ee51482 100644 --- a/Modules/expat/xmlrole.c +++ b/Modules/expat/xmlrole.c @@ -170,7 +170,14 @@ prolog1(PROLOG_STATE *state, case XML_TOK_COMMENT: return XML_ROLE_COMMENT; case XML_TOK_BOM: - return XML_ROLE_NONE; + /* This case can never arise. To reach this role function, the + * parse must have passed through prolog0 and therefore have had + * some form of input, even if only a space. At that point, a + * byte order mark is no longer a valid character (though + * technically it should be interpreted as a non-breaking space), + * so will be rejected by the tokenizing stages. + */ + return XML_ROLE_NONE; /* LCOV_EXCL_LINE */ case XML_TOK_DECL_OPEN: if (!XmlNameMatchesAscii(enc, ptr + 2 * MIN_BYTES_PER_CHAR(enc), @@ -1285,6 +1292,26 @@ declClose(PROLOG_STATE *state, return common(state, tok); } +/* This function will only be invoked if the internal logic of the + * parser has broken down. It is used in two cases: + * + * 1: When the XML prolog has been finished. At this point the + * processor (the parser level above these role handlers) should + * switch from prologProcessor to contentProcessor and reinitialise + * the handler function. + * + * 2: When an error has been detected (via common() below). At this + * point again the processor should be switched to errorProcessor, + * which will never call a handler. + * + * The result of this is that error() can only be called if the + * processor switch failed to happen, which is an internal error and + * therefore we shouldn't be able to provoke it simply by using the + * library. It is a necessary backstop, however, so we merely exclude + * it from the coverage statistics. + * + * LCOV_EXCL_START + */ static int PTRCALL error(PROLOG_STATE *UNUSED_P(state), int UNUSED_P(tok), @@ -1294,6 +1321,7 @@ error(PROLOG_STATE *UNUSED_P(state), { return XML_ROLE_NONE; } +/* LCOV_EXCL_STOP */ static int FASTCALL common(PROLOG_STATE *state, int tok) diff --git a/Modules/expat/xmltok.c b/Modules/expat/xmltok.c index cdf0720dd89..db4a5c8ca3e 100644 --- a/Modules/expat/xmltok.c +++ b/Modules/expat/xmltok.c @@ -1019,7 +1019,11 @@ streqci(const char *s1, const char *s2) if (ASCII_a <= c1 && c1 <= ASCII_z) c1 += ASCII_A - ASCII_a; if (ASCII_a <= c2 && c2 <= ASCII_z) - c2 += ASCII_A - ASCII_a; + /* The following line will never get executed. streqci() is + * only called from two places, both of which guarantee to put + * upper-case strings into s2. + */ + c2 += ASCII_A - ASCII_a; /* LCOV_EXCL_LINE */ if (c1 != c2) return 0; if (!c1) @@ -1291,7 +1295,7 @@ XmlUtf8Encode(int c, char *buf) }; if (c < 0) - return 0; + return 0; /* LCOV_EXCL_LINE: this case is always eliminated beforehand */ if (c < min2) { buf[0] = (char)(c | UTF8_cval1); return 1; @@ -1314,7 +1318,7 @@ XmlUtf8Encode(int c, char *buf) buf[3] = (char)((c & 0x3f) | 0x80); return 4; } - return 0; + return 0; /* LCOV_EXCL_LINE: this case too is eliminated before calling */ } int FASTCALL @@ -1465,6 +1469,9 @@ XmlInitUnknownEncoding(void *mem, else if (c < 0) { if (c < -4) return 0; + /* Multi-byte sequences need a converter function */ + if (!convert) + return 0; e->normal.type[i] = (unsigned char)(BT_LEAD2 - (c + 2)); e->utf8[i][0] = 0; e->utf16[i] = 0; diff --git a/Modules/expat/xmltok_impl.c b/Modules/expat/xmltok_impl.c index 5f779c0571b..4fa1ff679ce 100644 --- a/Modules/expat/xmltok_impl.c +++ b/Modules/expat/xmltok_impl.c @@ -1198,8 +1198,14 @@ PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr, const char *start; if (ptr >= end) return XML_TOK_NONE; - else if (! HAS_CHAR(enc, ptr, end)) - return XML_TOK_PARTIAL; + else if (! HAS_CHAR(enc, ptr, end)) { + /* This line cannot be executed. The incoming data has already + * been tokenized once, so incomplete characters like this have + * already been eliminated from the input. Retaining the paranoia + * check is still valuable, however. + */ + return XML_TOK_PARTIAL; /* LCOV_EXCL_LINE */ + } start = ptr; while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { @@ -1258,8 +1264,14 @@ PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr, const char *start; if (ptr >= end) return XML_TOK_NONE; - else if (! HAS_CHAR(enc, ptr, end)) - return XML_TOK_PARTIAL; + else if (! HAS_CHAR(enc, ptr, end)) { + /* This line cannot be executed. The incoming data has already + * been tokenized once, so incomplete characters like this have + * already been eliminated from the input. Retaining the paranoia + * check is still valuable, however. + */ + return XML_TOK_PARTIAL; /* LCOV_EXCL_LINE */ + } start = ptr; while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { @@ -1614,6 +1626,14 @@ PREFIX(predefinedEntityName)(const ENCODING *UNUSED_P(enc), const char *ptr, return 0; } +/* This function does not appear to be called from anywhere within the + * library code. It is used via the macro XmlSameName(), which is + * defined but never used. Since it appears in the encoding function + * table, removing it is not a thing to be undertaken lightly. For + * the moment, we simply exclude it from coverage tests. + * + * LCOV_EXCL_START + */ static int PTRCALL PREFIX(sameName)(const ENCODING *enc, const char *ptr1, const char *ptr2) { @@ -1677,14 +1697,21 @@ PREFIX(sameName)(const ENCODING *enc, const char *ptr1, const char *ptr2) } /* not reached */ } +/* LCOV_EXCL_STOP */ static int PTRCALL PREFIX(nameMatchesAscii)(const ENCODING *UNUSED_P(enc), const char *ptr1, const char *end1, const char *ptr2) { for (; *ptr2; ptr1 += MINBPC(enc), ptr2++) { - if (end1 - ptr1 < MINBPC(enc)) - return 0; + if (end1 - ptr1 < MINBPC(enc)) { + /* This line cannot be executed. THe incoming data has already + * been tokenized once, so imcomplete characters like this have + * already been eliminated from the input. Retaining the + * paranoia check is still valuable, however. + */ + return 0; /* LCOV_EXCL_LINE */ + } if (!CHAR_MATCHES(enc, ptr1, *ptr2)) return 0; } diff --git a/PC/VS9.0/_elementtree.vcproj b/PC/VS9.0/_elementtree.vcproj index f89409cc3ed..2915eb3eef0 100644 --- a/PC/VS9.0/_elementtree.vcproj +++ b/PC/VS9.0/_elementtree.vcproj @@ -595,6 +595,10 @@ > </File> <File + RelativePath="..\..\Modules\expat\loadlibrary.c" + > + </File> + <File RelativePath="..\..\Modules\expat\xmlparse.c" > </File> diff --git a/PC/VS9.0/pyexpat.vcproj b/PC/VS9.0/pyexpat.vcproj index 76b698f7402..7a2474e67cb 100644 --- a/PC/VS9.0/pyexpat.vcproj +++ b/PC/VS9.0/pyexpat.vcproj @@ -535,6 +535,10 @@ > </File> <File + RelativePath="..\..\Modules\expat\loadlibrary.c" + > + </File> + <File RelativePath="..\..\Modules\expat\xmlparse.c" > </File> diff --git a/PCbuild/_elementtree.vcxproj b/PCbuild/_elementtree.vcxproj index 511e26bb60d..1b024b9d854 100644 --- a/PCbuild/_elementtree.vcxproj +++ b/PCbuild/_elementtree.vcxproj @@ -87,6 +87,7 @@ </ItemGroup> <ItemGroup> <ClCompile Include="..\Modules\_elementtree.c" /> + <ClCompile Include="..\Modules\expat\loadlibrary.c" /> <ClCompile Include="..\Modules\expat\xmlparse.c" /> <ClCompile Include="..\Modules\expat\xmlrole.c" /> <ClCompile Include="..\Modules\expat\xmltok.c" /> diff --git a/PCbuild/_elementtree.vcxproj.filters b/PCbuild/_elementtree.vcxproj.filters index 6acdf35846a..4597ee521b3 100644 --- a/PCbuild/_elementtree.vcxproj.filters +++ b/PCbuild/_elementtree.vcxproj.filters @@ -33,6 +33,9 @@ <ClInclude Include="..\Modules\expat\latin1tab.h"> <Filter>Header Files</Filter> </ClInclude> + <ClInclude Include="..\Modules\expat\loadlibrary.c"> + <Filter>Header Files</Filter> + </ClInclude> <ClInclude Include="..\Modules\expat\macconfig.h"> <Filter>Header Files</Filter> </ClInclude> diff --git a/PCbuild/pyexpat.vcxproj b/PCbuild/pyexpat.vcxproj index 61a2697e116..e8be337c259 100644 --- a/PCbuild/pyexpat.vcxproj +++ b/PCbuild/pyexpat.vcxproj @@ -68,6 +68,7 @@ </ItemGroup> <ItemGroup> <ClCompile Include="..\Modules\pyexpat.c" /> + <ClCompile Include="..\Modules\expat\loadlibrary.c" /> <ClCompile Include="..\Modules\expat\xmlparse.c" /> <ClCompile Include="..\Modules\expat\xmlrole.c" /> <ClCompile Include="..\Modules\expat\xmltok.c" /> diff --git a/PCbuild/pyexpat.vcxproj.filters b/PCbuild/pyexpat.vcxproj.filters index f8d46026c9c..cb02847980c 100644 --- a/PCbuild/pyexpat.vcxproj.filters +++ b/PCbuild/pyexpat.vcxproj.filters @@ -20,6 +20,9 @@ <ClCompile Include="..\Modules\pyexpat.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\Modules\expat\loadlibrary.c"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="..\Modules\expat\xmlparse.c"> <Filter>Source Files</Filter> </ClCompile> diff --git a/setup.py b/setup.py index 54054c208a7..5503486ccbf 100644 --- a/setup.py +++ b/setup.py @@ -1492,6 +1492,9 @@ class db_found(Exception): pass expat_inc = [os.path.join(os.getcwd(), srcdir, 'Modules', 'expat')] define_macros = [ ('HAVE_EXPAT_CONFIG_H', '1'), + # bpo-30947: Python uses best available entropy sources to + # call XML_SetHashSalt(), expat entropy sources are not needed + ('XML_POOR_ENTROPY', '1'), ] expat_lib = [] expat_sources = ['expat/xmlparse.c',

1 0

bpo-30947: Update libexpat from 2.2.1 to 2.2.3 (#3106) (#3143)
by Victor Stinner 18 Aug '17

18 Aug '17

https://github.com/python/cpython/commit/83e37e16f3065086d721d4e62a3788e01d… commit: 83e37e16f3065086d721d4e62a3788e01db3431c branch: 3.6 author: Victor Stinner <victor.stinner(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-08-19T01:06:27+02:00 summary: bpo-30947: Update libexpat from 2.2.1 to 2.2.3 (#3106) (#3143) * bpo-30947: Update libexpat from 2.2.1 to 2.2.3 * Add NEWS entry * Add new loadlibrary.c * expat_external.h: restore include "pyexpatns.h" * PCbuild: add expat/loadlibrary.c * Define XML_POOR_ENTROPY to compile expat (cherry picked from commit 93d0cb58b4da2a88c56f472c6c19491cc7a390df) files: A Misc/NEWS.d/next/Security/2017-08-16-16-35-59.bpo-30947.iNMmm4.rst A Modules/expat/loadlibrary.c M Modules/expat/expat.h M Modules/expat/siphash.h M Modules/expat/xmlparse.c M Modules/expat/xmlrole.c M Modules/expat/xmltok.c M Modules/expat/xmltok_impl.c M PCbuild/_elementtree.vcxproj M PCbuild/_elementtree.vcxproj.filters M PCbuild/pyexpat.vcxproj M PCbuild/pyexpat.vcxproj.filters M setup.py diff --git a/Misc/NEWS.d/next/Security/2017-08-16-16-35-59.bpo-30947.iNMmm4.rst b/Misc/NEWS.d/next/Security/2017-08-16-16-35-59.bpo-30947.iNMmm4.rst new file mode 100644 index 00000000000..3caca9a79b4 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2017-08-16-16-35-59.bpo-30947.iNMmm4.rst @@ -0,0 +1,2 @@ +Upgrade libexpat embedded copy from version 2.2.1 to 2.2.3 to get security +fixes. diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index 28b0f954d41..7e5bbb7e393 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -24,7 +24,6 @@ extern "C" { struct XML_ParserStruct; typedef struct XML_ParserStruct *XML_Parser; -/* Should this be defined using stdbool.h when C99 is available? */ typedef unsigned char XML_Bool; #define XML_TRUE ((XML_Bool) 1) #define XML_FALSE ((XML_Bool) 0) @@ -1049,7 +1048,7 @@ XML_GetFeatureList(void); */ #define XML_MAJOR_VERSION 2 #define XML_MINOR_VERSION 2 -#define XML_MICRO_VERSION 1 +#define XML_MICRO_VERSION 3 #ifdef __cplusplus } diff --git a/Modules/expat/loadlibrary.c b/Modules/expat/loadlibrary.c new file mode 100644 index 00000000000..ffce868399b --- /dev/null +++ b/Modules/expat/loadlibrary.c @@ -0,0 +1,141 @@ +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 2016 - 2017, Steve Holme, <steve_holme(a)hotmail.com>. + * + * All rights reserved. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF + * THIRD PARTY RIGHTS. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH + * THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * Except as contained in this notice, the name of a copyright holder shall + * not be used in advertising or otherwise to promote the sale, use or other + * dealings in this Software without prior written authorization of the + * copyright holder. + * + ***************************************************************************/ + +#if defined(_WIN32) + +#include <windows.h> +#include <tchar.h> + + +HMODULE _Expat_LoadLibrary(LPCTSTR filename); + + +#if !defined(LOAD_WITH_ALTERED_SEARCH_PATH) +#define LOAD_WITH_ALTERED_SEARCH_PATH 0x00000008 +#endif + +#if !defined(LOAD_LIBRARY_SEARCH_SYSTEM32) +#define LOAD_LIBRARY_SEARCH_SYSTEM32 0x00000800 +#endif + +/* We use our own typedef here since some headers might lack these */ +typedef HMODULE (APIENTRY *LOADLIBRARYEX_FN)(LPCTSTR, HANDLE, DWORD); + +/* See function definitions in winbase.h */ +#ifdef UNICODE +# ifdef _WIN32_WCE +# define LOADLIBARYEX L"LoadLibraryExW" +# else +# define LOADLIBARYEX "LoadLibraryExW" +# endif +#else +# define LOADLIBARYEX "LoadLibraryExA" +#endif + + +/* + * _Expat_LoadLibrary() + * + * This is used to dynamically load DLLs using the most secure method available + * for the version of Windows that we are running on. + * + * Parameters: + * + * filename [in] - The filename or full path of the DLL to load. If only the + * filename is passed then the DLL will be loaded from the + * Windows system directory. + * + * Returns the handle of the module on success; otherwise NULL. + */ +HMODULE _Expat_LoadLibrary(LPCTSTR filename) +{ + HMODULE hModule = NULL; + LOADLIBRARYEX_FN pLoadLibraryEx = NULL; + + /* Get a handle to kernel32 so we can access it's functions at runtime */ + HMODULE hKernel32 = GetModuleHandle(TEXT("kernel32")); + if(!hKernel32) + return NULL; + + /* Attempt to find LoadLibraryEx() which is only available on Windows 2000 + and above */ + pLoadLibraryEx = (LOADLIBRARYEX_FN) GetProcAddress(hKernel32, LOADLIBARYEX); + + /* Detect if there's already a path in the filename and load the library if + there is. Note: Both back slashes and forward slashes have been supported + since the earlier days of DOS at an API level although they are not + supported by command prompt */ + if(_tcspbrk(filename, TEXT("\\/"))) { + /** !checksrc! disable BANNEDFUNC 1 **/ + hModule = pLoadLibraryEx ? + pLoadLibraryEx(filename, NULL, LOAD_WITH_ALTERED_SEARCH_PATH) : + LoadLibrary(filename); + } + /* Detect if KB2533623 is installed, as LOAD_LIBARY_SEARCH_SYSTEM32 is only + supported on Windows Vista, Windows Server 2008, Windows 7 and Windows + Server 2008 R2 with this patch or natively on Windows 8 and above */ + else if(pLoadLibraryEx && GetProcAddress(hKernel32, "AddDllDirectory")) { + /* Load the DLL from the Windows system directory */ + hModule = pLoadLibraryEx(filename, NULL, LOAD_LIBRARY_SEARCH_SYSTEM32); + } + else { + /* Attempt to get the Windows system path */ + UINT systemdirlen = GetSystemDirectory(NULL, 0); + if(systemdirlen) { + /* Allocate space for the full DLL path (Room for the null terminator + is included in systemdirlen) */ + size_t filenamelen = _tcslen(filename); + TCHAR *path = malloc(sizeof(TCHAR) * (systemdirlen + 1 + filenamelen)); + if(path && GetSystemDirectory(path, systemdirlen)) { + /* Calculate the full DLL path */ + _tcscpy(path + _tcslen(path), TEXT("\\")); + _tcscpy(path + _tcslen(path), filename); + + /* Load the DLL from the Windows system directory */ + /** !checksrc! disable BANNEDFUNC 1 **/ + hModule = pLoadLibraryEx ? + pLoadLibraryEx(path, NULL, LOAD_WITH_ALTERED_SEARCH_PATH) : + LoadLibrary(path); + + } + free(path); + } + } + + return hModule; +} + +#else /* defined(_WIN32) */ + +/* ISO C requires a translation unit to contain at least one declaration + [-Wempty-translation-unit] */ +typedef int _TRANSLATION_UNIT_LOAD_LIBRARY_C_NOT_EMTPY; + +#endif /* defined(_WIN32) */ diff --git a/Modules/expat/siphash.h b/Modules/expat/siphash.h index 23b56d2ae48..581872df7b4 100644 --- a/Modules/expat/siphash.h +++ b/Modules/expat/siphash.h @@ -2,9 +2,8 @@ * siphash.h - SipHash-2-4 in a single header file * -------------------------------------------------------------------------- * Derived by William Ahern from the reference implementation[1] published[2] - * by Jean-Philippe Aumasson and Daniel J. Berstein. Licensed in kind. * by Jean-Philippe Aumasson and Daniel J. Berstein. - * Minimal changes by Sebastian Pipping on top, details below. + * Minimal changes by Sebastian Pipping and Victor Stinner on top, see below. * Licensed under the CC0 Public Domain Dedication license. * * 1. https://www.131002.net/siphash/siphash24.c @@ -12,14 +11,25 @@ * -------------------------------------------------------------------------- * HISTORY: * - * 2017-06-10 (Sebastian Pipping) + * 2017-07-25 (Vadim Zeitlin) + * - Fix use of SIPHASH_MAIN macro + * + * 2017-07-05 (Sebastian Pipping) + * - Use _SIP_ULL macro to not require a C++11 compiler if compiled as C++ + * - Add const qualifiers at two places + * - Ensure <=80 characters line length (assuming tab width 4) + * + * 2017-06-23 (Victor Stinner) + * - Address Win64 compile warnings + * + * 2017-06-18 (Sebastian Pipping) * - Clarify license note in the header * - Address C89 issues: * - Stop using inline keyword (and let compiler decide) - * - Turn integer suffix ULL to UL * - Replace _Bool by int * - Turn macro siphash24 into a function * - Address invalid conversion (void pointer) by explicit cast + * - Address lack of stdint.h for Visual Studio 2003 to 2008 * - Always expose sip24_valid (for self-tests) * * 2012-11-04 - Born. (William Ahern) @@ -76,7 +86,23 @@ #define SIPHASH_H #include <stddef.h> /* size_t */ -#include <stdint.h> /* uint64_t uint32_t uint8_t */ + +#if defined(_WIN32) && defined(_MSC_VER) && (_MSC_VER < 1600) + /* For vs2003/7.1 up to vs2008/9.0; _MSC_VER 1600 is vs2010/10.0 */ + typedef unsigned __int8 uint8_t; + typedef unsigned __int32 uint32_t; + typedef unsigned __int64 uint64_t; +#else + #include <stdint.h> /* uint64_t uint32_t uint8_t */ +#endif + + +/* + * Workaround to not require a C++11 compiler for using ULL suffix + * if this code is included and compiled as C++; related GCC warning is: + * warning: use of C++11 long long integer constant [-Wlong-long] + */ +#define _SIP_ULL(high, low) (((uint64_t)high << 32) | low) #define SIP_ROTL(x, b) (uint64_t)(((x) << (b)) | ( (x) >> (64 - (b)))) @@ -158,11 +184,12 @@ static void sip_round(struct siphash *H, const int rounds) { } /* sip_round() */ -static struct siphash *sip24_init(struct siphash *H, const struct sipkey *key) { - H->v0 = 0x736f6d6570736575UL ^ key->k[0]; - H->v1 = 0x646f72616e646f6dUL ^ key->k[1]; - H->v2 = 0x6c7967656e657261UL ^ key->k[0]; - H->v3 = 0x7465646279746573UL ^ key->k[1]; +static struct siphash *sip24_init(struct siphash *H, + const struct sipkey *key) { + H->v0 = _SIP_ULL(0x736f6d65U, 0x70736575U) ^ key->k[0]; + H->v1 = _SIP_ULL(0x646f7261U, 0x6e646f6dU) ^ key->k[1]; + H->v2 = _SIP_ULL(0x6c796765U, 0x6e657261U) ^ key->k[0]; + H->v3 = _SIP_ULL(0x74656462U, 0x79746573U) ^ key->k[1]; H->p = H->buf; H->c = 0; @@ -173,7 +200,8 @@ static struct siphash *sip24_init(struct siphash *H, const struct sipkey *key) { #define sip_endof(a) (&(a)[sizeof (a) / sizeof *(a)]) -static struct siphash *sip24_update(struct siphash *H, const void *src, size_t len) { +static struct siphash *sip24_update(struct siphash *H, const void *src, + size_t len) { const unsigned char *p = (const unsigned char *)src, *pe = p + len; uint64_t m; @@ -198,7 +226,7 @@ static struct siphash *sip24_update(struct siphash *H, const void *src, size_t l static uint64_t sip24_final(struct siphash *H) { - char left = H->p - H->buf; + const char left = (char)(H->p - H->buf); uint64_t b = (H->c + left) << 56; switch (left) { @@ -222,7 +250,8 @@ static uint64_t sip24_final(struct siphash *H) { } /* sip24_final() */ -static uint64_t siphash24(const void *src, size_t len, const struct sipkey *key) { +static uint64_t siphash24(const void *src, size_t len, + const struct sipkey *key) { struct siphash state = SIPHASH_INITIALIZER; return sip24_final(sip24_update(sip24_init(&state, key), src, len)); } /* siphash24() */ @@ -310,10 +339,11 @@ static int sip24_valid(void) { struct sipkey k; size_t i; - sip_tokey(&k, "\000\001\002\003\004\005\006\007\010\011\012\013\014\015\016\017"); + sip_tokey(&k, "\000\001\002\003\004\005\006\007\010\011" + "\012\013\014\015\016\017"); for (i = 0; i < sizeof in; ++i) { - in[i] = i; + in[i] = (unsigned char)i; if (siphash24(in, i, &k) != SIP_U8TO64_LE(vectors[i])) return 0; @@ -323,12 +353,12 @@ static int sip24_valid(void) { } /* sip24_valid() */ -#if SIPHASH_MAIN +#ifdef SIPHASH_MAIN #include <stdio.h> int main(void) { - int ok = sip24_valid(); + const int ok = sip24_valid(); if (ok) puts("OK"); diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index daec151e232..b703e61a040 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -1,10 +1,12 @@ /* Copyright (c) 1998, 1999, 2000 Thai Open Source Software Center Ltd See the file COPYING for copying permission. - 77fea421d361dca90041d0040ecf1dca651167fadf2af79e990e35168d70d933 (2.2.1+) + 101bfd65d1ff3d1511cf6671e6aae65f82cd97df6f4da137d46d510731830ad9 (2.2.3+) */ -#define _GNU_SOURCE 1 /* syscall prototype */ +#if !defined(_GNU_SOURCE) +# define _GNU_SOURCE 1 /* syscall prototype */ +#endif #include <stddef.h> #include <string.h> /* memset(), memcpy() */ @@ -19,6 +21,8 @@ #include <sys/time.h> /* gettimeofday() */ #include <sys/types.h> /* getpid() */ #include <unistd.h> /* getpid() */ +#include <fcntl.h> /* O_RDONLY */ +#include <errno.h> #endif #define XML_BUILDING_EXPAT 1 @@ -33,6 +37,57 @@ #include "expat.h" #include "siphash.h" +#if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) +# if defined(HAVE_GETRANDOM) +# include <sys/random.h> /* getrandom */ +# else +# include <unistd.h> /* syscall */ +# include <sys/syscall.h> /* SYS_getrandom */ +# endif +# if ! defined(GRND_NONBLOCK) +# define GRND_NONBLOCK 0x0001 +# endif /* defined(GRND_NONBLOCK) */ +#endif /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */ + +#if defined(HAVE_LIBBSD) \ + && (defined(HAVE_ARC4RANDOM_BUF) || defined(HAVE_ARC4RANDOM)) +# include <bsd/stdlib.h> +#endif + +#if defined(_WIN32) && !defined(LOAD_LIBRARY_SEARCH_SYSTEM32) +# define LOAD_LIBRARY_SEARCH_SYSTEM32 0x00000800 +#endif + +#if !defined(HAVE_GETRANDOM) && !defined(HAVE_SYSCALL_GETRANDOM) \ + && !defined(HAVE_ARC4RANDOM_BUF) && !defined(HAVE_ARC4RANDOM) \ + && !defined(XML_DEV_URANDOM) \ + && !defined(_WIN32) \ + && !defined(XML_POOR_ENTROPY) +# error \ + You do not have support for any sources of high quality entropy \ + enabled. For end user security, that is probably not what you want. \ + \ + Your options include: \ + * Linux + glibc >=2.25 (getrandom): HAVE_GETRANDOM, \ + * Linux + glibc <2.25 (syscall SYS_getrandom): HAVE_SYSCALL_GETRANDOM, \ + * BSD / macOS >=10.7 (arc4random_buf): HAVE_ARC4RANDOM_BUF, \ + * BSD / macOS <10.7 (arc4random): HAVE_ARC4RANDOM, \ + * libbsd (arc4random_buf): HAVE_ARC4RANDOM_BUF + HAVE_LIBBSD, \ + * libbsd (arc4random): HAVE_ARC4RANDOM + HAVE_LIBBSD, \ + * Linux / BSD / macOS (/dev/urandom): XML_DEV_URANDOM \ + * Windows (RtlGenRandom): _WIN32. \ + \ + If insist on not using any of these, bypass this error by defining \ + XML_POOR_ENTROPY; you have been warned. \ + \ + For CMake, one way to pass the define is: \ + cmake -DCMAKE_C_FLAGS="-pipe -O2 -DHAVE_SYSCALL_GETRANDOM" . \ + \ + If you have reasons to patch this detection code away or need changes \ + to the build system, please open a bug. Thank you! +#endif + + #ifdef XML_UNICODE #define XML_ENCODE_MAX XML_UTF16_ENCODE_MAX #define XmlConvert XmlUtf16Convert @@ -436,6 +491,9 @@ static ELEMENT_TYPE * getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr, const char *end); +static XML_Char *copyString(const XML_Char *s, + const XML_Memory_Handling_Suite *memsuite); + static unsigned long generate_hash_secret_salt(XML_Parser parser); static XML_Bool startParsing(XML_Parser parser); @@ -696,21 +754,13 @@ static const XML_Char implicitContext[] = { #if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) -# include <errno.h> - -# if defined(HAVE_GETRANDOM) -# include <sys/random.h> /* getrandom */ -# else -# include <unistd.h> /* syscall */ -# include <sys/syscall.h> /* SYS_getrandom */ -# endif /* Obtain entropy on Linux 3.17+ */ static int -writeRandomBytes_getrandom(void * target, size_t count) { +writeRandomBytes_getrandom_nonblock(void * target, size_t count) { int success = 0; /* full count bytes written? */ size_t bytesWrittenTotal = 0; - const unsigned int getrandomFlags = 0; + const unsigned int getrandomFlags = GRND_NONBLOCK; do { void * const currentTarget = (void*)((char*)target + bytesWrittenTotal); @@ -728,7 +778,7 @@ writeRandomBytes_getrandom(void * target, size_t count) { if (bytesWrittenTotal >= count) success = 1; } - } while (! success && (errno == EINTR || errno == EAGAIN)); + } while (! success && (errno == EINTR)); return success; } @@ -736,12 +786,67 @@ writeRandomBytes_getrandom(void * target, size_t count) { #endif /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */ +#if ! defined(_WIN32) && defined(XML_DEV_URANDOM) + +/* Extract entropy from /dev/urandom */ +static int +writeRandomBytes_dev_urandom(void * target, size_t count) { + int success = 0; /* full count bytes written? */ + size_t bytesWrittenTotal = 0; + + const int fd = open("/dev/urandom", O_RDONLY); + if (fd < 0) { + return 0; + } + + do { + void * const currentTarget = (void*)((char*)target + bytesWrittenTotal); + const size_t bytesToWrite = count - bytesWrittenTotal; + + const ssize_t bytesWrittenMore = read(fd, currentTarget, bytesToWrite); + + if (bytesWrittenMore > 0) { + bytesWrittenTotal += bytesWrittenMore; + if (bytesWrittenTotal >= count) + success = 1; + } + } while (! success && (errno == EINTR)); + + close(fd); + return success; +} + +#endif /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */ + + +#if defined(HAVE_ARC4RANDOM) + +static void +writeRandomBytes_arc4random(void * target, size_t count) { + size_t bytesWrittenTotal = 0; + + while (bytesWrittenTotal < count) { + const uint32_t random32 = arc4random(); + size_t i = 0; + + for (; (i < sizeof(random32)) && (bytesWrittenTotal < count); + i++, bytesWrittenTotal++) { + const uint8_t random8 = (uint8_t)(random32 >> (i * 8)); + ((uint8_t *)target)[bytesWrittenTotal] = random8; + } + } +} + +#endif /* defined(HAVE_ARC4RANDOM) */ + + #ifdef _WIN32 typedef BOOLEAN (APIENTRY *RTLGENRANDOM_FUNC)(PVOID, ULONG); +HMODULE _Expat_LoadLibrary(LPCTSTR filename); /* see loadlibrary.c */ /* Obtain entropy on Windows XP / Windows Server 2003 and later. - * Hint on RtlGenRandom and the following article from libsodioum. + * Hint on RtlGenRandom and the following article from libsodium. * * Michael Howard: Cryptographically Secure Random number on Windows without using CryptoAPI * https://blogs.msdn.microsoft.com/michael_howard/2005/01/14/cryptographicall… @@ -749,7 +854,7 @@ typedef BOOLEAN (APIENTRY *RTLGENRANDOM_FUNC)(PVOID, ULONG); static int writeRandomBytes_RtlGenRandom(void * target, size_t count) { int success = 0; /* full count bytes written? */ - const HMODULE advapi32 = LoadLibrary("ADVAPI32.DLL"); + const HMODULE advapi32 = _Expat_LoadLibrary(TEXT("ADVAPI32.DLL")); if (advapi32) { const RTLGENRANDOM_FUNC RtlGenRandom @@ -768,6 +873,8 @@ writeRandomBytes_RtlGenRandom(void * target, size_t count) { #endif /* _WIN32 */ +#if ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) + static unsigned long gather_time_entropy(void) { @@ -780,16 +887,20 @@ gather_time_entropy(void) int gettimeofday_res; gettimeofday_res = gettimeofday(&tv, NULL); + +#if defined(NDEBUG) + (void)gettimeofday_res; +#else assert (gettimeofday_res == 0); +#endif /* defined(NDEBUG) */ /* Microseconds time is <20 bits entropy */ return tv.tv_usec; #endif } -#if defined(HAVE_ARC4RANDOM_BUF) && defined(HAVE_LIBBSD) -# include <bsd/stdlib.h> -#endif +#endif /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */ + static unsigned long ENTROPY_DEBUG(const char * label, unsigned long entropy) { @@ -808,10 +919,12 @@ generate_hash_secret_salt(XML_Parser parser) { unsigned long entropy; (void)parser; -#if defined(HAVE_ARC4RANDOM_BUF) || defined(__CloudABI__) - (void)gather_time_entropy; +#if defined(HAVE_ARC4RANDOM_BUF) arc4random_buf(&entropy, sizeof(entropy)); return ENTROPY_DEBUG("arc4random_buf", entropy); +#elif defined(HAVE_ARC4RANDOM) + writeRandomBytes_arc4random((void *)&entropy, sizeof(entropy)); + return ENTROPY_DEBUG("arc4random", entropy); #else /* Try high quality providers first .. */ #ifdef _WIN32 @@ -819,10 +932,15 @@ generate_hash_secret_salt(XML_Parser parser) return ENTROPY_DEBUG("RtlGenRandom", entropy); } #elif defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) - if (writeRandomBytes_getrandom((void *)&entropy, sizeof(entropy))) { + if (writeRandomBytes_getrandom_nonblock((void *)&entropy, sizeof(entropy))) { return ENTROPY_DEBUG("getrandom", entropy); } #endif +#if ! defined(_WIN32) && defined(XML_DEV_URANDOM) + if (writeRandomBytes_dev_urandom((void *)&entropy, sizeof(entropy))) { + return ENTROPY_DEBUG("/dev/urandom", entropy); + } +#endif /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */ /* .. and self-made low quality for backup: */ /* Process ID is 0 bits entropy if attacker has local access */ @@ -833,7 +951,7 @@ generate_hash_secret_salt(XML_Parser parser) return ENTROPY_DEBUG("fallback(4)", entropy * 2147483647); } else { return ENTROPY_DEBUG("fallback(8)", - entropy * (unsigned long)2305843009213693951); + entropy * (unsigned long)2305843009213693951ULL); } #endif } @@ -962,6 +1080,8 @@ parserCreate(const XML_Char *encodingName, nsAttsVersion = 0; nsAttsPower = 0; + protocolEncodingName = NULL; + poolInit(&tempPool, &(parser->m_mem)); poolInit(&temp2Pool, &(parser->m_mem)); parserInit(parser, encodingName); @@ -988,9 +1108,9 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { processor = prologInitProcessor; XmlPrologStateInit(&prologState); - protocolEncodingName = (encodingName != NULL - ? poolCopyString(&tempPool, encodingName) - : NULL); + if (encodingName != NULL) { + protocolEncodingName = copyString(encodingName, &(parser->m_mem)); + } curBase = NULL; XmlInitEncoding(&initEncoding, &encoding, 0); userData = NULL; @@ -1103,6 +1223,8 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) unknownEncodingRelease(unknownEncodingData); poolClear(&tempPool); poolClear(&temp2Pool); + FREE((void *)protocolEncodingName); + protocolEncodingName = NULL; parserInit(parser, encodingName); dtdReset(_dtd, &parser->m_mem); return XML_TRUE; @@ -1119,10 +1241,16 @@ XML_SetEncoding(XML_Parser parser, const XML_Char *encodingName) */ if (ps_parsing == XML_PARSING || ps_parsing == XML_SUSPENDED) return XML_STATUS_ERROR; + + /* Get rid of any previous encoding name */ + FREE((void *)protocolEncodingName); + if (encodingName == NULL) + /* No new encoding name */ protocolEncodingName = NULL; else { - protocolEncodingName = poolCopyString(&tempPool, encodingName); + /* Copy the new encoding name into allocated memory */ + protocolEncodingName = copyString(encodingName, &(parser->m_mem)); if (!protocolEncodingName) return XML_STATUS_ERROR; } @@ -1357,6 +1485,7 @@ XML_ParserFree(XML_Parser parser) destroyBindings(inheritedBindings, parser); poolDestroy(&tempPool); poolDestroy(&temp2Pool); + FREE((void *)protocolEncodingName); #ifdef XML_DTD /* external parameter entity parsers share the DTD structure parser->m_dtd with the root parser, so we must not destroy it @@ -1748,7 +1877,8 @@ enum XML_Status XMLCALL XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { if ((parser == NULL) || (len < 0) || ((s == NULL) && (len != 0))) { - errorCode = XML_ERROR_INVALID_ARGUMENT; + if (parser != NULL) + parser->m_errorCode = XML_ERROR_INVALID_ARGUMENT; return XML_STATUS_ERROR; } switch (ps_parsing) { @@ -1783,9 +1913,22 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) if (errorCode == XML_ERROR_NONE) { switch (ps_parsing) { case XML_SUSPENDED: + /* It is hard to be certain, but it seems that this case + * cannot occur. This code is cleaning up a previous parse + * with no new data (since len == 0). Changing the parsing + * state requires getting to execute a handler function, and + * there doesn't seem to be an opportunity for that while in + * this circumstance. + * + * Given the uncertainty, we retain the code but exclude it + * from coverage tests. + * + * LCOV_EXCL_START + */ XmlUpdatePosition(encoding, positionPtr, bufferPtr, &position); positionPtr = bufferPtr; return XML_STATUS_SUSPENDED; + /* LCOV_EXCL_STOP */ case XML_INITIALIZED: case XML_PARSING: ps_parsing = XML_FINISHED; @@ -2974,9 +3117,17 @@ doContent(XML_Parser parser, return XML_ERROR_NO_MEMORY; break; default: + /* All of the tokens produced by XmlContentTok() have their own + * explicit cases, so this default is not strictly necessary. + * However it is a useful safety net, so we retain the code and + * simply exclude it from the coverage tests. + * + * LCOV_EXCL_START + */ if (defaultHandler) reportDefault(parser, enc, s, next); break; + /* LCOV_EXCL_STOP */ } *eventPP = s = next; switch (ps_parsing) { @@ -3067,13 +3218,17 @@ storeAtts(XML_Parser parser, const ENCODING *enc, #endif attsSize = n + nDefaultAtts + INIT_ATTS_SIZE; temp = (ATTRIBUTE *)REALLOC((void *)atts, attsSize * sizeof(ATTRIBUTE)); - if (temp == NULL) + if (temp == NULL) { + attsSize = oldAttsSize; return XML_ERROR_NO_MEMORY; + } atts = temp; #ifdef XML_ATTR_INFO temp2 = (XML_AttrInfo *)REALLOC((void *)attInfo, attsSize * sizeof(XML_AttrInfo)); - if (temp2 == NULL) + if (temp2 == NULL) { + attsSize = oldAttsSize; return XML_ERROR_NO_MEMORY; + } attInfo = temp2; #endif if (n > oldAttsSize) @@ -3210,6 +3365,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, int j; /* hash table index */ unsigned long version = nsAttsVersion; int nsAttsSize = (int)1 << nsAttsPower; + unsigned char oldNsAttsPower = nsAttsPower; /* size of hash table must be at least 2 * (# of prefixed attributes) */ if ((nPrefixes << 1) >> nsAttsPower) { /* true for nsAttsPower = 0 */ NS_ATT *temp; @@ -3219,8 +3375,11 @@ storeAtts(XML_Parser parser, const ENCODING *enc, nsAttsPower = 3; nsAttsSize = (int)1 << nsAttsPower; temp = (NS_ATT *)REALLOC(nsAtts, nsAttsSize * sizeof(NS_ATT)); - if (!temp) + if (!temp) { + /* Restore actual size of memory in nsAtts */ + nsAttsPower = oldNsAttsPower; return XML_ERROR_NO_MEMORY; + } nsAtts = temp; version = 0; /* force re-initialization of nsAtts hash table */ } @@ -3247,8 +3406,23 @@ storeAtts(XML_Parser parser, const ENCODING *enc, ((XML_Char *)s)[-1] = 0; /* clear flag */ id = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, s, 0); - if (!id || !id->prefix) - return XML_ERROR_NO_MEMORY; + if (!id || !id->prefix) { + /* This code is walking through the appAtts array, dealing + * with (in this case) a prefixed attribute name. To be in + * the array, the attribute must have already been bound, so + * has to have passed through the hash table lookup once + * already. That implies that an entry for it already + * exists, so the lookup above will return a pointer to + * already allocated memory. There is no opportunaity for + * the allocator to fail, so the condition above cannot be + * fulfilled. + * + * Since it is difficult to be certain that the above + * analysis is complete, we retain the test and merely + * remove the code from coverage tests. + */ + return XML_ERROR_NO_MEMORY; /* LCOV_EXCL_LINE */ + } b = id->prefix->binding; if (!b) return XML_ERROR_UNBOUND_PREFIX; @@ -3625,8 +3799,16 @@ doCdataSection(XML_Parser parser, } return XML_ERROR_UNCLOSED_CDATA_SECTION; default: + /* Every token returned by XmlCdataSectionTok() has its own + * explicit case, so this default case will never be executed. + * We retain it as a safety net and exclude it from the coverage + * statistics. + * + * LCOV_EXCL_START + */ *eventPP = next; return XML_ERROR_UNEXPECTED_STATE; + /* LCOV_EXCL_STOP */ } *eventPP = s = next; @@ -3686,8 +3868,20 @@ doIgnoreSection(XML_Parser parser, eventEndPP = &eventEndPtr; } else { + /* It's not entirely clear, but it seems the following two lines + * of code cannot be executed. The only occasions on which 'enc' + * is not 'parser->m_encoding' are when this function is called + * from the internal entity processing, and IGNORE sections are an + * error in internal entities. + * + * Since it really isn't clear that this is true, we keep the code + * and just remove it from our coverage tests. + * + * LCOV_EXCL_START + */ eventPP = &(openInternalEntities->internalEventPtr); eventEndPP = &(openInternalEntities->internalEventEndPtr); + /* LCOV_EXCL_STOP */ } *eventPP = s; *startPtr = NULL; @@ -3720,8 +3914,16 @@ doIgnoreSection(XML_Parser parser, } return XML_ERROR_SYNTAX; /* XML_ERROR_UNCLOSED_IGNORE_SECTION */ default: + /* All of the tokens that XmlIgnoreSectionTok() returns have + * explicit cases to handle them, so this default case is never + * executed. We keep it as a safety net anyway, and remove it + * from our test coverage statistics. + * + * LCOV_EXCL_START + */ *eventPP = next; return XML_ERROR_UNEXPECTED_STATE; + /* LCOV_EXCL_STOP */ } /* not reached */ } @@ -3734,6 +3936,7 @@ initializeEncoding(XML_Parser parser) const char *s; #ifdef XML_UNICODE char encodingBuf[128]; + /* See comments abount `protoclEncodingName` in parserInit() */ if (!protocolEncodingName) s = NULL; else { @@ -3817,7 +4020,14 @@ processXmlDecl(XML_Parser parser, int isGeneralTextEntity, reportDefault(parser, encoding, s, next); if (protocolEncodingName == NULL) { if (newEncoding) { - if (newEncoding->minBytesPerChar != encoding->minBytesPerChar) { + /* Check that the specified encoding does not conflict with what + * the parser has already deduced. Do we have the same number + * of bytes in the smallest representation of a character? If + * this is UTF-16, is it the same endianness? + */ + if (newEncoding->minBytesPerChar != encoding->minBytesPerChar + || (newEncoding->minBytesPerChar == 2 && + newEncoding != encoding)) { eventPtr = encodingName; return XML_ERROR_INCORRECT_ENCODING; } @@ -3962,15 +4172,14 @@ entityValueInitProcessor(XML_Parser parser, result = processXmlDecl(parser, 0, start, next); if (result != XML_ERROR_NONE) return result; - switch (ps_parsing) { - case XML_SUSPENDED: - *nextPtr = next; - return XML_ERROR_NONE; - case XML_FINISHED: + /* At this point, ps_parsing cannot be XML_SUSPENDED. For that + * to happen, a parameter entity parsing handler must have + * attempted to suspend the parser, which fails and raises an + * error. The parser can be aborted, but can't be suspended. + */ + if (ps_parsing == XML_FINISHED) return XML_ERROR_ABORTED; - default: - *nextPtr = next; - } + *nextPtr = next; /* stop scanning for text declaration - we found one */ processor = entityValueProcessor; return entityValueProcessor(parser, next, end, nextPtr); @@ -4293,8 +4502,14 @@ doProlog(XML_Parser parser, &dtd->paramEntities, externalSubsetName, sizeof(ENTITY)); - if (!entity) - return XML_ERROR_NO_MEMORY; + if (!entity) { + /* The external subset name "#" will have already been + * inserted into the hash table at the start of the + * external entity parsing, so no allocation will happen + * and lookup() cannot fail. + */ + return XML_ERROR_NO_MEMORY; /* LCOV_EXCL_LINE */ + } if (useForeignDTD) entity->base = curBase; dtd->paramEntityRead = XML_FALSE; @@ -4773,8 +4988,10 @@ doProlog(XML_Parser parser, if (prologState.level >= groupSize) { if (groupSize) { char *temp = (char *)REALLOC(groupConnector, groupSize *= 2); - if (temp == NULL) + if (temp == NULL) { + groupSize /= 2; return XML_ERROR_NO_MEMORY; + } groupConnector = temp; if (dtd->scaffIndex) { int *temp = (int *)REALLOC(dtd->scaffIndex, @@ -4786,8 +5003,10 @@ doProlog(XML_Parser parser, } else { groupConnector = (char *)MALLOC(groupSize = 32); - if (!groupConnector) + if (!groupConnector) { + groupSize = 0; return XML_ERROR_NO_MEMORY; + } } } groupConnector[prologState.level] = 0; @@ -4850,8 +5069,29 @@ doProlog(XML_Parser parser, : !dtd->hasParamEntityRefs)) { if (!entity) return XML_ERROR_UNDEFINED_ENTITY; - else if (!entity->is_internal) - return XML_ERROR_ENTITY_DECLARED_IN_PE; + else if (!entity->is_internal) { + /* It's hard to exhaustively search the code to be sure, + * but there doesn't seem to be a way of executing the + * following line. There are two cases: + * + * If 'standalone' is false, the DTD must have no + * parameter entities or we wouldn't have passed the outer + * 'if' statement. That measn the only entity in the hash + * table is the external subset name "#" which cannot be + * given as a parameter entity name in XML syntax, so the + * lookup must have returned NULL and we don't even reach + * the test for an internal entity. + * + * If 'standalone' is true, it does not seem to be + * possible to create entities taking this code path that + * are not internal entities, so fail the test above. + * + * Because this analysis is very uncertain, the code is + * being left in place and merely removed from the + * coverage test statistics. + */ + return XML_ERROR_ENTITY_DECLARED_IN_PE; /* LCOV_EXCL_LINE */ + } } else if (!entity) { dtd->keepProcessing = dtd->standalone; @@ -5323,11 +5563,15 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, && (poolLength(pool) == 0 || poolLastChar(pool) == 0x20)) break; n = XmlEncode(n, (ICHAR *)buf); - if (!n) { - if (enc == encoding) - eventPtr = ptr; - return XML_ERROR_BAD_CHAR_REF; - } + /* The XmlEncode() functions can never return 0 here. That + * error return happens if the code point passed in is either + * negative or greater than or equal to 0x110000. The + * XmlCharRefNumber() functions will all return a number + * strictly less than 0x110000 or a negative value if an error + * occurred. The negative value is intercepted above, so + * XmlEncode() is never passed a value it might return an + * error for. + */ for (i = 0; i < n; i++) { if (!poolAppendChar(pool, buf[i])) return XML_ERROR_NO_MEMORY; @@ -5401,8 +5645,26 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, break; } if (entity->open) { - if (enc == encoding) - eventPtr = ptr; + if (enc == encoding) { + /* It does not appear that this line can be executed. + * + * The "if (entity->open)" check catches recursive entity + * definitions. In order to be called with an open + * entity, it must have gone through this code before and + * been through the recursive call to + * appendAttributeValue() some lines below. That call + * sets the local encoding ("enc") to the parser's + * internal encoding (internal_utf8 or internal_utf16), + * which can never be the same as the principle encoding. + * It doesn't appear there is another code path that gets + * here with entity->open being TRUE. + * + * Since it is not certain that this logic is watertight, + * we keep the line and merely exclude it from coverage + * tests. + */ + eventPtr = ptr; /* LCOV_EXCL_LINE */ + } return XML_ERROR_RECURSIVE_ENTITY_REF; } if (entity->notation) { @@ -5429,9 +5691,21 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, } break; default: + /* The only token returned by XmlAttributeValueTok() that does + * not have an explicit case here is XML_TOK_PARTIAL_CHAR. + * Getting that would require an entity name to contain an + * incomplete XML character (e.g. \xE2\x82); however previous + * tokenisers will have already recognised and rejected such + * names before XmlAttributeValueTok() gets a look-in. This + * default case should be retained as a safety net, but the code + * excluded from coverage tests. + * + * LCOV_EXCL_START + */ if (enc == encoding) eventPtr = ptr; return XML_ERROR_UNEXPECTED_STATE; + /* LCOV_EXCL_STOP */ } ptr = next; } @@ -5564,12 +5838,15 @@ storeEntityValue(XML_Parser parser, goto endEntityValue; } n = XmlEncode(n, (ICHAR *)buf); - if (!n) { - if (enc == encoding) - eventPtr = entityTextPtr; - result = XML_ERROR_BAD_CHAR_REF; - goto endEntityValue; - } + /* The XmlEncode() functions can never return 0 here. That + * error return happens if the code point passed in is either + * negative or greater than or equal to 0x110000. The + * XmlCharRefNumber() functions will all return a number + * strictly less than 0x110000 or a negative value if an error + * occurred. The negative value is intercepted above, so + * XmlEncode() is never passed a value it might return an + * error for. + */ for (i = 0; i < n; i++) { if (pool->end == pool->ptr && !poolGrow(pool)) { result = XML_ERROR_NO_MEMORY; @@ -5590,10 +5867,18 @@ storeEntityValue(XML_Parser parser, result = XML_ERROR_INVALID_TOKEN; goto endEntityValue; default: + /* This default case should be unnecessary -- all the tokens + * that XmlEntityValueTok() can return have their own explicit + * cases -- but should be retained for safety. We do however + * exclude it from the coverage statistics. + * + * LCOV_EXCL_START + */ if (enc == encoding) eventPtr = entityTextPtr; result = XML_ERROR_UNEXPECTED_STATE; goto endEntityValue; + /* LCOV_EXCL_STOP */ } entityTextPtr = next; } @@ -5691,8 +5976,25 @@ reportDefault(XML_Parser parser, const ENCODING *enc, eventEndPP = &eventEndPtr; } else { + /* To get here, two things must be true; the parser must be + * using a character encoding that is not the same as the + * encoding passed in, and the encoding passed in must need + * conversion to the internal format (UTF-8 unless XML_UNICODE + * is defined). The only occasions on which the encoding passed + * in is not the same as the parser's encoding are when it is + * the internal encoding (e.g. a previously defined parameter + * entity, already converted to internal format). This by + * definition doesn't need conversion, so the whole branch never + * gets executed. + * + * For safety's sake we don't delete these lines and merely + * exclude them from coverage statistics. + * + * LCOV_EXCL_START + */ eventPP = &(openInternalEntities->internalEventPtr); eventEndPP = &(openInternalEntities->internalEventEndPtr); + /* LCOV_EXCL_STOP */ } do { ICHAR *dataPtr = (ICHAR *)dataBuf; @@ -5861,9 +6163,30 @@ getContext(XML_Parser parser) len = dtd->defaultPrefix.binding->uriLen; if (namespaceSeparator) len--; - for (i = 0; i < len; i++) - if (!poolAppendChar(&tempPool, dtd->defaultPrefix.binding->uri[i])) - return NULL; + for (i = 0; i < len; i++) { + if (!poolAppendChar(&tempPool, dtd->defaultPrefix.binding->uri[i])) { + /* Because of memory caching, I don't believe this line can be + * executed. + * + * This is part of a loop copying the default prefix binding + * URI into the parser's temporary string pool. Previously, + * that URI was copied into the same string pool, with a + * terminating NUL character, as part of setContext(). When + * the pool was cleared, that leaves a block definitely big + * enough to hold the URI on the free block list of the pool. + * The URI copy in getContext() therefore cannot run out of + * memory. + * + * If the pool is used between the setContext() and + * getContext() calls, the worst it can do is leave a bigger + * block on the front of the free list. Given that this is + * all somewhat inobvious and program logic can be changed, we + * don't delete the line but we do exclude it from the test + * coverage statistics. + */ + return NULL; /* LCOV_EXCL_LINE */ + } + } needSep = XML_TRUE; } @@ -5875,8 +6198,15 @@ getContext(XML_Parser parser) PREFIX *prefix = (PREFIX *)hashTableIterNext(&iter); if (!prefix) break; - if (!prefix->binding) - continue; + if (!prefix->binding) { + /* This test appears to be (justifiable) paranoia. There does + * not seem to be a way of injecting a prefix without a binding + * that doesn't get errored long before this function is called. + * The test should remain for safety's sake, so we instead + * exclude the following line from the coverage statistics. + */ + continue; /* LCOV_EXCL_LINE */ + } if (needSep && !poolAppendChar(&tempPool, CONTEXT_SEP)) return NULL; for (s = prefix->name; *s; s++) @@ -6547,8 +6877,20 @@ poolCopyString(STRING_POOL *pool, const XML_Char *s) static const XML_Char * poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n) { - if (!pool->ptr && !poolGrow(pool)) - return NULL; + if (!pool->ptr && !poolGrow(pool)) { + /* The following line is unreachable given the current usage of + * poolCopyStringN(). Currently it is called from exactly one + * place to copy the text of a simple general entity. By that + * point, the name of the entity is already stored in the pool, so + * pool->ptr cannot be NULL. + * + * If poolCopyStringN() is used elsewhere as it well might be, + * this line may well become executable again. Regardless, this + * sort of check shouldn't be removed lightly, so we just exclude + * it from the coverage statistics. + */ + return NULL; /* LCOV_EXCL_LINE */ + } for (; n > 0; --n, s++) { if (!poolAppendChar(pool, *s)) return NULL; @@ -6641,8 +6983,19 @@ poolGrow(STRING_POOL *pool) int blockSize = (int)((unsigned)(pool->end - pool->start)*2U); size_t bytesToAllocate; - if (blockSize < 0) - return XML_FALSE; + // NOTE: Needs to be calculated prior to calling `realloc` + // to avoid dangling pointers: + const ptrdiff_t offsetInsideBlock = pool->ptr - pool->start; + + if (blockSize < 0) { + /* This condition traps a situation where either more than + * INT_MAX/2 bytes have already been allocated. This isn't + * readily testable, since it is unlikely that an average + * machine will have that much memory, so we exclude it from the + * coverage statistics. + */ + return XML_FALSE; /* LCOV_EXCL_LINE */ + } bytesToAllocate = poolBytesToAllocateFor(blockSize); if (bytesToAllocate == 0) @@ -6654,7 +7007,7 @@ poolGrow(STRING_POOL *pool) return XML_FALSE; pool->blocks = temp; pool->blocks->size = blockSize; - pool->ptr = pool->blocks->s + (pool->ptr - pool->start); + pool->ptr = pool->blocks->s + offsetInsideBlock; pool->start = pool->blocks->s; pool->end = pool->start + blockSize; } @@ -6663,8 +7016,18 @@ poolGrow(STRING_POOL *pool) int blockSize = (int)(pool->end - pool->start); size_t bytesToAllocate; - if (blockSize < 0) - return XML_FALSE; + if (blockSize < 0) { + /* This condition traps a situation where either more than + * INT_MAX bytes have already been allocated (which is prevented + * by various pieces of program logic, not least this one, never + * mind the unlikelihood of actually having that much memory) or + * the pool control fields have been corrupted (which could + * conceivably happen in an extremely buggy user handler + * function). Either way it isn't readily testable, so we + * exclude it from the coverage statistics. + */ + return XML_FALSE; /* LCOV_EXCL_LINE */ + } if (blockSize < INIT_BLOCK_SIZE) blockSize = INIT_BLOCK_SIZE; @@ -6827,3 +7190,26 @@ getElementType(XML_Parser parser, } return ret; } + +static XML_Char * +copyString(const XML_Char *s, + const XML_Memory_Handling_Suite *memsuite) +{ + int charsRequired = 0; + XML_Char *result; + + /* First determine how long the string is */ + while (s[charsRequired] != 0) { + charsRequired++; + } + /* Include the terminator */ + charsRequired++; + + /* Now allocate space for the copy */ + result = memsuite->malloc_fcn(charsRequired * sizeof(XML_Char)); + if (result == NULL) + return NULL; + /* Copy the original into place */ + memcpy(result, s, charsRequired * sizeof(XML_Char)); + return result; +} diff --git a/Modules/expat/xmlrole.c b/Modules/expat/xmlrole.c index a7c56302796..c809ee51482 100644 --- a/Modules/expat/xmlrole.c +++ b/Modules/expat/xmlrole.c @@ -170,7 +170,14 @@ prolog1(PROLOG_STATE *state, case XML_TOK_COMMENT: return XML_ROLE_COMMENT; case XML_TOK_BOM: - return XML_ROLE_NONE; + /* This case can never arise. To reach this role function, the + * parse must have passed through prolog0 and therefore have had + * some form of input, even if only a space. At that point, a + * byte order mark is no longer a valid character (though + * technically it should be interpreted as a non-breaking space), + * so will be rejected by the tokenizing stages. + */ + return XML_ROLE_NONE; /* LCOV_EXCL_LINE */ case XML_TOK_DECL_OPEN: if (!XmlNameMatchesAscii(enc, ptr + 2 * MIN_BYTES_PER_CHAR(enc), @@ -1285,6 +1292,26 @@ declClose(PROLOG_STATE *state, return common(state, tok); } +/* This function will only be invoked if the internal logic of the + * parser has broken down. It is used in two cases: + * + * 1: When the XML prolog has been finished. At this point the + * processor (the parser level above these role handlers) should + * switch from prologProcessor to contentProcessor and reinitialise + * the handler function. + * + * 2: When an error has been detected (via common() below). At this + * point again the processor should be switched to errorProcessor, + * which will never call a handler. + * + * The result of this is that error() can only be called if the + * processor switch failed to happen, which is an internal error and + * therefore we shouldn't be able to provoke it simply by using the + * library. It is a necessary backstop, however, so we merely exclude + * it from the coverage statistics. + * + * LCOV_EXCL_START + */ static int PTRCALL error(PROLOG_STATE *UNUSED_P(state), int UNUSED_P(tok), @@ -1294,6 +1321,7 @@ error(PROLOG_STATE *UNUSED_P(state), { return XML_ROLE_NONE; } +/* LCOV_EXCL_STOP */ static int FASTCALL common(PROLOG_STATE *state, int tok) diff --git a/Modules/expat/xmltok.c b/Modules/expat/xmltok.c index cdf0720dd89..db4a5c8ca3e 100644 --- a/Modules/expat/xmltok.c +++ b/Modules/expat/xmltok.c @@ -1019,7 +1019,11 @@ streqci(const char *s1, const char *s2) if (ASCII_a <= c1 && c1 <= ASCII_z) c1 += ASCII_A - ASCII_a; if (ASCII_a <= c2 && c2 <= ASCII_z) - c2 += ASCII_A - ASCII_a; + /* The following line will never get executed. streqci() is + * only called from two places, both of which guarantee to put + * upper-case strings into s2. + */ + c2 += ASCII_A - ASCII_a; /* LCOV_EXCL_LINE */ if (c1 != c2) return 0; if (!c1) @@ -1291,7 +1295,7 @@ XmlUtf8Encode(int c, char *buf) }; if (c < 0) - return 0; + return 0; /* LCOV_EXCL_LINE: this case is always eliminated beforehand */ if (c < min2) { buf[0] = (char)(c | UTF8_cval1); return 1; @@ -1314,7 +1318,7 @@ XmlUtf8Encode(int c, char *buf) buf[3] = (char)((c & 0x3f) | 0x80); return 4; } - return 0; + return 0; /* LCOV_EXCL_LINE: this case too is eliminated before calling */ } int FASTCALL @@ -1465,6 +1469,9 @@ XmlInitUnknownEncoding(void *mem, else if (c < 0) { if (c < -4) return 0; + /* Multi-byte sequences need a converter function */ + if (!convert) + return 0; e->normal.type[i] = (unsigned char)(BT_LEAD2 - (c + 2)); e->utf8[i][0] = 0; e->utf16[i] = 0; diff --git a/Modules/expat/xmltok_impl.c b/Modules/expat/xmltok_impl.c index 5f779c0571b..4fa1ff679ce 100644 --- a/Modules/expat/xmltok_impl.c +++ b/Modules/expat/xmltok_impl.c @@ -1198,8 +1198,14 @@ PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr, const char *start; if (ptr >= end) return XML_TOK_NONE; - else if (! HAS_CHAR(enc, ptr, end)) - return XML_TOK_PARTIAL; + else if (! HAS_CHAR(enc, ptr, end)) { + /* This line cannot be executed. The incoming data has already + * been tokenized once, so incomplete characters like this have + * already been eliminated from the input. Retaining the paranoia + * check is still valuable, however. + */ + return XML_TOK_PARTIAL; /* LCOV_EXCL_LINE */ + } start = ptr; while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { @@ -1258,8 +1264,14 @@ PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr, const char *start; if (ptr >= end) return XML_TOK_NONE; - else if (! HAS_CHAR(enc, ptr, end)) - return XML_TOK_PARTIAL; + else if (! HAS_CHAR(enc, ptr, end)) { + /* This line cannot be executed. The incoming data has already + * been tokenized once, so incomplete characters like this have + * already been eliminated from the input. Retaining the paranoia + * check is still valuable, however. + */ + return XML_TOK_PARTIAL; /* LCOV_EXCL_LINE */ + } start = ptr; while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { @@ -1614,6 +1626,14 @@ PREFIX(predefinedEntityName)(const ENCODING *UNUSED_P(enc), const char *ptr, return 0; } +/* This function does not appear to be called from anywhere within the + * library code. It is used via the macro XmlSameName(), which is + * defined but never used. Since it appears in the encoding function + * table, removing it is not a thing to be undertaken lightly. For + * the moment, we simply exclude it from coverage tests. + * + * LCOV_EXCL_START + */ static int PTRCALL PREFIX(sameName)(const ENCODING *enc, const char *ptr1, const char *ptr2) { @@ -1677,14 +1697,21 @@ PREFIX(sameName)(const ENCODING *enc, const char *ptr1, const char *ptr2) } /* not reached */ } +/* LCOV_EXCL_STOP */ static int PTRCALL PREFIX(nameMatchesAscii)(const ENCODING *UNUSED_P(enc), const char *ptr1, const char *end1, const char *ptr2) { for (; *ptr2; ptr1 += MINBPC(enc), ptr2++) { - if (end1 - ptr1 < MINBPC(enc)) - return 0; + if (end1 - ptr1 < MINBPC(enc)) { + /* This line cannot be executed. THe incoming data has already + * been tokenized once, so imcomplete characters like this have + * already been eliminated from the input. Retaining the + * paranoia check is still valuable, however. + */ + return 0; /* LCOV_EXCL_LINE */ + } if (!CHAR_MATCHES(enc, ptr1, *ptr2)) return 0; } diff --git a/PCbuild/_elementtree.vcxproj b/PCbuild/_elementtree.vcxproj index 639ba479db5..0db61f20818 100644 --- a/PCbuild/_elementtree.vcxproj +++ b/PCbuild/_elementtree.vcxproj @@ -87,6 +87,7 @@ </ItemGroup> <ItemGroup> <ClCompile Include="..\Modules\_elementtree.c" /> + <ClCompile Include="..\Modules\expat\loadlibrary.c" /> <ClCompile Include="..\Modules\expat\xmlparse.c" /> <ClCompile Include="..\Modules\expat\xmlrole.c" /> <ClCompile Include="..\Modules\expat\xmltok.c" /> diff --git a/PCbuild/_elementtree.vcxproj.filters b/PCbuild/_elementtree.vcxproj.filters index ee78295fa94..4597ee521b3 100644 --- a/PCbuild/_elementtree.vcxproj.filters +++ b/PCbuild/_elementtree.vcxproj.filters @@ -33,6 +33,9 @@ <ClInclude Include="..\Modules\expat\latin1tab.h"> <Filter>Header Files</Filter> </ClInclude> + <ClInclude Include="..\Modules\expat\loadlibrary.c"> + <Filter>Header Files</Filter> + </ClInclude> <ClInclude Include="..\Modules\expat\macconfig.h"> <Filter>Header Files</Filter> </ClInclude> @@ -69,4 +72,4 @@ <Filter>Source Files</Filter> </ClCompile> </ItemGroup> -</Project> \ No newline at end of file +</Project> diff --git a/PCbuild/pyexpat.vcxproj b/PCbuild/pyexpat.vcxproj index a2763f02063..51ca69e95d7 100644 --- a/PCbuild/pyexpat.vcxproj +++ b/PCbuild/pyexpat.vcxproj @@ -68,6 +68,7 @@ </ItemGroup> <ItemGroup> <ClCompile Include="..\Modules\pyexpat.c" /> + <ClCompile Include="..\Modules\expat\loadlibrary.c" /> <ClCompile Include="..\Modules\expat\xmlparse.c" /> <ClCompile Include="..\Modules\expat\xmlrole.c" /> <ClCompile Include="..\Modules\expat\xmltok.c" /> @@ -84,4 +85,4 @@ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <ImportGroup Label="ExtensionTargets"> </ImportGroup> -</Project> \ No newline at end of file +</Project> diff --git a/PCbuild/pyexpat.vcxproj.filters b/PCbuild/pyexpat.vcxproj.filters index abf362fb0bf..cb02847980c 100644 --- a/PCbuild/pyexpat.vcxproj.filters +++ b/PCbuild/pyexpat.vcxproj.filters @@ -20,6 +20,9 @@ <ClCompile Include="..\Modules\pyexpat.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\Modules\expat\loadlibrary.c"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="..\Modules\expat\xmlparse.c"> <Filter>Source Files</Filter> </ClCompile> @@ -30,4 +33,4 @@ <Filter>Source Files</Filter> </ClCompile> </ItemGroup> -</Project> \ No newline at end of file +</Project> diff --git a/setup.py b/setup.py index fe477974bd0..1cd15039970 100644 --- a/setup.py +++ b/setup.py @@ -1503,6 +1503,9 @@ class db_found(Exception): pass expat_inc = [os.path.join(os.getcwd(), srcdir, 'Modules', 'expat')] define_macros = [ ('HAVE_EXPAT_CONFIG_H', '1'), + # bpo-30947: Python uses best available entropy sources to + # call XML_SetHashSalt(), expat entropy sources are not needed + ('XML_POOR_ENTROPY', '1'), ] expat_lib = [] expat_sources = ['expat/xmlparse.c',

1 0

gitignore: add more rules from master for Windows (#3148)
by Victor Stinner 18 Aug '17

18 Aug '17

https://github.com/python/cpython/commit/5e006aa05fe8228ebfb5405005a253dba2… commit: 5e006aa05fe8228ebfb5405005a253dba2238379 branch: 2.7 author: Victor Stinner <victor.stinner(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-08-19T00:50:15+02:00 summary: gitignore: add more rules from master for Windows (#3148) files: M .gitignore diff --git a/.gitignore b/.gitignore index cf5db7681f3..a8c73d4b14b 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,9 @@ PC/*/*.suo PC/*/Win32-temp-* PC/*/x64-temp-* PC/*/amd64 +PCbuild/*.user +PCbuild/*.suo +PCbuild/*.*sdf PCbuild/*.bsc PCbuild/*.dll PCbuild/*.exe @@ -55,6 +58,9 @@ PCbuild/*.pdb PCbuild/Win32-temp-* PCbuild/*.VC.db PCbuild/*.VC.opendb +PCbuild/amd64/ +PCbuild/obj/ +PCbuild/win32/ Parser/pgen Parser/pgen.stamp autom4te.cache @@ -70,6 +76,7 @@ platform pybuilddir.txt pyconfig.h python$ +python.bat python.exe python*-gdb.py tags

1 0

bpo-31206: IDLE: Factor HighPage class from ConfigDialog (#3141)
by Terry Jan Reedy 18 Aug '17

18 Aug '17

https://github.com/python/cpython/commit/a32e40561a24de373d1c5a437a8aa32975… commit: a32e40561a24de373d1c5a437a8aa329758ba8e4 branch: master author: Cheryl Sabella <cheryl.sabella(a)gmail.com> committer: Terry Jan Reedy <tjreedy(a)udel.edu> date: 2017-08-18T18:34:55-04:00 summary: bpo-31206: IDLE: Factor HighPage class from ConfigDialog (#3141) This is the first half of a patch similar to the one for for bpo-31205. It is being split into 2 PRs to avoid what happened with PR-3096 -- an incomprehensible diff that could not be cleanly backported to 3.6. This half copies several methods of ConfigDialog and turns them into a new class. files: A Misc/NEWS.d/next/IDLE/2017-08-18-14-13-42.bpo-31206.F1-tKK.rst M Lib/idlelib/configdialog.py diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index a36bf83ad0f..587f16ec9df 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -1290,6 +1290,649 @@ def var_changed_space_num(self, *params): changes.add_option('main', 'Indent', 'num-spaces', value) +class HighPage(Frame): + + def __init__(self, master): + super().__init__(master) + self.cd = master.master + self.create_page_highlight() + self.load_theme_cfg() + + def create_page_highlight(self): + """Return frame of widgets for Highlighting tab. + + Enable users to provisionally change foreground and background + colors applied to textual tags. Color mappings are stored in + complete listings called themes. Built-in themes in + idlelib/config-highlight.def are fixed as far as the dialog is + concerned. Any theme can be used as the base for a new custom + theme, stored in .idlerc/config-highlight.cfg. + + Function load_theme_cfg() initializes tk variables and theme + lists and calls paint_theme_sample() and set_highlight_target() + for the current theme. Radiobuttons builtin_theme_on and + custom_theme_on toggle var theme_source, which controls if the + current set of colors are from a builtin or custom theme. + DynOptionMenus builtinlist and customlist contain lists of the + builtin and custom themes, respectively, and the current item + from each list is stored in vars builtin_name and custom_name. + + Function paint_theme_sample() applies the colors from the theme + to the tags in text widget highlight_sample and then invokes + set_color_sample(). Function set_highlight_target() sets the state + of the radiobuttons fg_on and bg_on based on the tag and it also + invokes set_color_sample(). + + Function set_color_sample() sets the background color for the frame + holding the color selector. This provides a larger visual of the + color for the current tag and plane (foreground/background). + + Note: set_color_sample() is called from many places and is often + called more than once when a change is made. It is invoked when + foreground or background is selected (radiobuttons), from + paint_theme_sample() (theme is changed or load_cfg is called), and + from set_highlight_target() (target tag is changed or load_cfg called). + + Button delete_custom invokes delete_custom() to delete + a custom theme from idleConf.userCfg['highlight'] and changes. + Button save_custom invokes save_as_new_theme() which calls + get_new_theme_name() and create_new() to save a custom theme + and its colors to idleConf.userCfg['highlight']. + + Radiobuttons fg_on and bg_on toggle var fg_bg_toggle to control + if the current selected color for a tag is for the foreground or + background. + + DynOptionMenu targetlist contains a readable description of the + tags applied to Python source within IDLE. Selecting one of the + tags from this list populates highlight_target, which has a callback + function set_highlight_target(). + + Text widget highlight_sample displays a block of text (which is + mock Python code) in which is embedded the defined tags and reflects + the color attributes of the current theme and changes for those tags. + Mouse button 1 allows for selection of a tag and updates + highlight_target with that tag value. + + Note: The font in highlight_sample is set through the config in + the fonts tab. + + In other words, a tag can be selected either from targetlist or + by clicking on the sample text within highlight_sample. The + plane (foreground/background) is selected via the radiobutton. + Together, these two (tag and plane) control what color is + shown in set_color_sample() for the current theme. Button set_color + invokes get_color() which displays a ColorChooser to change the + color for the selected tag/plane. If a new color is picked, + it will be saved to changes and the highlight_sample and + frame background will be updated. + + Tk Variables: + color: Color of selected target. + builtin_name: Menu variable for built-in theme. + custom_name: Menu variable for custom theme. + fg_bg_toggle: Toggle for foreground/background color. + Note: this has no callback. + theme_source: Selector for built-in or custom theme. + highlight_target: Menu variable for the highlight tag target. + + Instance Data Attributes: + theme_elements: Dictionary of tags for text highlighting. + The key is the display name and the value is a tuple of + (tag name, display sort order). + + Methods [attachment]: + load_theme_cfg: Load current highlight colors. + get_color: Invoke colorchooser [button_set_color]. + set_color_sample_binding: Call set_color_sample [fg_bg_toggle]. + set_highlight_target: set fg_bg_toggle, set_color_sample(). + set_color_sample: Set frame background to target. + on_new_color_set: Set new color and add option. + paint_theme_sample: Recolor sample. + get_new_theme_name: Get from popup. + create_new: Combine theme with changes and save. + save_as_new_theme: Save [button_save_custom]. + set_theme_type: Command for [theme_source]. + delete_custom: Activate default [button_delete_custom]. + save_new: Save to userCfg['theme'] (is function). + + Widgets of highlights page frame: (*) widgets bound to self + frame_custom: LabelFrame + (*)highlight_sample: Text + (*)frame_color_set: Frame + (*)button_set_color: Button + (*)targetlist: DynOptionMenu - highlight_target + frame_fg_bg_toggle: Frame + (*)fg_on: Radiobutton - fg_bg_toggle + (*)bg_on: Radiobutton - fg_bg_toggle + (*)button_save_custom: Button + frame_theme: LabelFrame + theme_type_title: Label + (*)builtin_theme_on: Radiobutton - theme_source + (*)custom_theme_on: Radiobutton - theme_source + (*)builtinlist: DynOptionMenu - builtin_name + (*)customlist: DynOptionMenu - custom_name + (*)button_delete_custom: Button + (*)theme_message: Label + """ + self.theme_elements = { + 'Normal Text': ('normal', '00'), + 'Python Keywords': ('keyword', '01'), + 'Python Definitions': ('definition', '02'), + 'Python Builtins': ('builtin', '03'), + 'Python Comments': ('comment', '04'), + 'Python Strings': ('string', '05'), + 'Selected Text': ('hilite', '06'), + 'Found Text': ('hit', '07'), + 'Cursor': ('cursor', '08'), + 'Editor Breakpoint': ('break', '09'), + 'Shell Normal Text': ('console', '10'), + 'Shell Error Text': ('error', '11'), + 'Shell Stdout Text': ('stdout', '12'), + 'Shell Stderr Text': ('stderr', '13'), + } + self.builtin_name = tracers.add( + StringVar(self), self.var_changed_builtin_name) + self.custom_name = tracers.add( + StringVar(self), self.var_changed_custom_name) + self.fg_bg_toggle = BooleanVar(self) + self.color = tracers.add( + StringVar(self), self.var_changed_color) + self.theme_source = tracers.add( + BooleanVar(self), self.var_changed_theme_source) + self.highlight_target = tracers.add( + StringVar(self), self.var_changed_highlight_target) + + # Create widgets: + # body frame and section frames. + frame_custom = LabelFrame(self, borderwidth=2, relief=GROOVE, + text=' Custom Highlighting ') + frame_theme = LabelFrame(self, borderwidth=2, relief=GROOVE, + text=' Highlighting Theme ') + # frame_custom. + text = self.highlight_sample = Text( + frame_custom, relief=SOLID, borderwidth=1, + font=('courier', 12, ''), cursor='hand2', width=21, height=13, + takefocus=FALSE, highlightthickness=0, wrap=NONE) + text.bind('<Double-Button-1>', lambda e: 'break') + text.bind('<B1-Motion>', lambda e: 'break') + text_and_tags=(('\n', 'normal'), + ('#you can click here', 'comment'), ('\n', 'normal'), + ('#to choose items', 'comment'), ('\n', 'normal'), + ('def', 'keyword'), (' ', 'normal'), + ('func', 'definition'), ('(param):\n ', 'normal'), + ('"""string"""', 'string'), ('\n var0 = ', 'normal'), + ("'string'", 'string'), ('\n var1 = ', 'normal'), + ("'selected'", 'hilite'), ('\n var2 = ', 'normal'), + ("'found'", 'hit'), ('\n var3 = ', 'normal'), + ('list', 'builtin'), ('(', 'normal'), + ('None', 'keyword'), (')\n', 'normal'), + (' breakpoint("line")', 'break'), ('\n\n', 'normal'), + (' error ', 'error'), (' ', 'normal'), + ('cursor |', 'cursor'), ('\n ', 'normal'), + ('shell', 'console'), (' ', 'normal'), + ('stdout', 'stdout'), (' ', 'normal'), + ('stderr', 'stderr'), ('\n\n', 'normal')) + for texttag in text_and_tags: + text.insert(END, texttag[0], texttag[1]) + for element in self.theme_elements: + def tem(event, elem=element): + event.widget.winfo_toplevel().highlight_target.set(elem) + text.tag_bind( + self.theme_elements[element][0], '<ButtonPress-1>', tem) + text['state'] = DISABLED + self.frame_color_set = Frame(frame_custom, relief=SOLID, borderwidth=1) + frame_fg_bg_toggle = Frame(frame_custom) + self.button_set_color = Button( + self.frame_color_set, text='Choose Color for :', + command=self.get_color, highlightthickness=0) + self.targetlist = DynOptionMenu( + self.frame_color_set, self.highlight_target, None, + highlightthickness=0) #, command=self.set_highlight_targetBinding + self.fg_on = Radiobutton( + frame_fg_bg_toggle, variable=self.fg_bg_toggle, value=1, + text='Foreground', command=self.set_color_sample_binding) + self.bg_on = Radiobutton( + frame_fg_bg_toggle, variable=self.fg_bg_toggle, value=0, + text='Background', command=self.set_color_sample_binding) + self.fg_bg_toggle.set(1) + self.button_save_custom = Button( + frame_custom, text='Save as New Custom Theme', + command=self.save_as_new_theme) + # frame_theme. + theme_type_title = Label(frame_theme, text='Select : ') + self.builtin_theme_on = Radiobutton( + frame_theme, variable=self.theme_source, value=1, + command=self.set_theme_type, text='a Built-in Theme') + self.custom_theme_on = Radiobutton( + frame_theme, variable=self.theme_source, value=0, + command=self.set_theme_type, text='a Custom Theme') + self.builtinlist = DynOptionMenu( + frame_theme, self.builtin_name, None, command=None) + self.customlist = DynOptionMenu( + frame_theme, self.custom_name, None, command=None) + self.button_delete_custom = Button( + frame_theme, text='Delete Custom Theme', + command=self.delete_custom) + self.theme_message = Label(frame_theme, bd=2) + + # Pack widgets: + # body. + frame_custom.pack(side=LEFT, padx=5, pady=5, expand=TRUE, fill=BOTH) + frame_theme.pack(side=LEFT, padx=5, pady=5, fill=Y) + # frame_custom. + self.frame_color_set.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=X) + frame_fg_bg_toggle.pack(side=TOP, padx=5, pady=0) + self.highlight_sample.pack( + side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) + self.button_set_color.pack(side=TOP, expand=TRUE, fill=X, padx=8, pady=4) + self.targetlist.pack(side=TOP, expand=TRUE, fill=X, padx=8, pady=3) + self.fg_on.pack(side=LEFT, anchor=E) + self.bg_on.pack(side=RIGHT, anchor=W) + self.button_save_custom.pack(side=BOTTOM, fill=X, padx=5, pady=5) + # frame_theme. + theme_type_title.pack(side=TOP, anchor=W, padx=5, pady=5) + self.builtin_theme_on.pack(side=TOP, anchor=W, padx=5) + self.custom_theme_on.pack(side=TOP, anchor=W, padx=5, pady=2) + self.builtinlist.pack(side=TOP, fill=X, padx=5, pady=5) + self.customlist.pack(side=TOP, fill=X, anchor=W, padx=5, pady=5) + self.button_delete_custom.pack(side=TOP, fill=X, padx=5, pady=5) + self.theme_message.pack(side=TOP, fill=X, pady=5) + + def load_theme_cfg(self): + """Load current configuration settings for the theme options. + + Based on the theme_source toggle, the theme is set as + either builtin or custom and the initial widget values + reflect the current settings from idleConf. + + Attributes updated: + theme_source: Set from idleConf. + builtinlist: List of default themes from idleConf. + customlist: List of custom themes from idleConf. + custom_theme_on: Disabled if there are no custom themes. + custom_theme: Message with additional information. + targetlist: Create menu from self.theme_elements. + + Methods: + set_theme_type + paint_theme_sample + set_highlight_target + """ + # Set current theme type radiobutton. + self.theme_source.set(idleConf.GetOption( + 'main', 'Theme', 'default', type='bool', default=1)) + # Set current theme. + current_option = idleConf.CurrentTheme() + # Load available theme option menus. + if self.theme_source.get(): # Default theme selected. + item_list = idleConf.GetSectionList('default', 'highlight') + item_list.sort() + self.builtinlist.SetMenu(item_list, current_option) + item_list = idleConf.GetSectionList('user', 'highlight') + item_list.sort() + if not item_list: + self.custom_theme_on['state'] = DISABLED + self.custom_name.set('- no custom themes -') + else: + self.customlist.SetMenu(item_list, item_list[0]) + else: # User theme selected. + item_list = idleConf.GetSectionList('user', 'highlight') + item_list.sort() + self.customlist.SetMenu(item_list, current_option) + item_list = idleConf.GetSectionList('default', 'highlight') + item_list.sort() + self.builtinlist.SetMenu(item_list, item_list[0]) + self.set_theme_type() + # Load theme element option menu. + theme_names = list(self.theme_elements.keys()) + theme_names.sort(key=lambda x: self.theme_elements[x][1]) + self.targetlist.SetMenu(theme_names, theme_names[0]) + self.paint_theme_sample() + self.set_highlight_target() + + def var_changed_builtin_name(self, *params): + """Process new builtin theme selection. + + Add the changed theme's name to the changed_items and recreate + the sample with the values from the selected theme. + """ + old_themes = ('IDLE Classic', 'IDLE New') + value = self.builtin_name.get() + if value not in old_themes: + if idleConf.GetOption('main', 'Theme', 'name') not in old_themes: + changes.add_option('main', 'Theme', 'name', old_themes[0]) + changes.add_option('main', 'Theme', 'name2', value) + self.theme_message['text'] = 'New theme, see Help' + self.theme_message['fg'] = '#500000' + else: + changes.add_option('main', 'Theme', 'name', value) + changes.add_option('main', 'Theme', 'name2', '') + self.theme_message['text'] = '' + self.theme_message['fg'] = 'black' + self.paint_theme_sample() + + def var_changed_custom_name(self, *params): + """Process new custom theme selection. + + If a new custom theme is selected, add the name to the + changed_items and apply the theme to the sample. + """ + value = self.custom_name.get() + if value != '- no custom themes -': + changes.add_option('main', 'Theme', 'name', value) + self.paint_theme_sample() + + def var_changed_theme_source(self, *params): + """Process toggle between builtin and custom theme. + + Update the default toggle value and apply the newly + selected theme type. + """ + value = self.theme_source.get() + changes.add_option('main', 'Theme', 'default', value) + if value: + self.var_changed_builtin_name() + else: + self.var_changed_custom_name() + + def var_changed_color(self, *params): + "Process change to color choice." + self.on_new_color_set() + + def var_changed_highlight_target(self, *params): + "Process selection of new target tag for highlighting." + self.set_highlight_target() + + def set_theme_type(self): + """Set available screen options based on builtin or custom theme. + + Attributes accessed: + theme_source + + Attributes updated: + builtinlist + customlist + button_delete_custom + custom_theme_on + + Called from: + handler for builtin_theme_on and custom_theme_on + delete_custom + create_new + load_theme_cfg + """ + if self.theme_source.get(): + self.builtinlist['state'] = NORMAL + self.customlist['state'] = DISABLED + self.button_delete_custom['state'] = DISABLED + else: + self.builtinlist['state'] = DISABLED + self.custom_theme_on['state'] = NORMAL + self.customlist['state'] = NORMAL + self.button_delete_custom['state'] = NORMAL + + def get_color(self): + """Handle button to select a new color for the target tag. + + If a new color is selected while using a builtin theme, a + name must be supplied to create a custom theme. + + Attributes accessed: + highlight_target + frame_color_set + theme_source + + Attributes updated: + color + + Methods: + get_new_theme_name + create_new + """ + target = self.highlight_target.get() + prev_color = self.frame_color_set.cget('bg') + rgbTuplet, color_string = tkColorChooser.askcolor( + parent=self, title='Pick new color for : '+target, + initialcolor=prev_color) + if color_string and (color_string != prev_color): + # User didn't cancel and they chose a new color. + if self.theme_source.get(): # Current theme is a built-in. + message = ('Your changes will be saved as a new Custom Theme. ' + 'Enter a name for your new Custom Theme below.') + new_theme = self.get_new_theme_name(message) + if not new_theme: # User cancelled custom theme creation. + return + else: # Create new custom theme based on previously active theme. + self.create_new(new_theme) + self.color.set(color_string) + else: # Current theme is user defined. + self.color.set(color_string) + + def on_new_color_set(self): + "Display sample of new color selection on the dialog." + new_color = self.color.get() + self.frame_color_set['bg'] = new_color # Set sample. + plane = 'foreground' if self.fg_bg_toggle.get() else 'background' + sample_element = self.theme_elements[self.highlight_target.get()][0] + self.highlight_sample.tag_config(sample_element, **{plane: new_color}) + theme = self.custom_name.get() + theme_element = sample_element + '-' + plane + changes.add_option('highlight', theme, theme_element, new_color) + + def get_new_theme_name(self, message): + "Return name of new theme from query popup." + used_names = (idleConf.GetSectionList('user', 'highlight') + + idleConf.GetSectionList('default', 'highlight')) + new_theme = SectionName( + self, 'New Custom Theme', message, used_names).result + return new_theme + + def save_as_new_theme(self): + """Prompt for new theme name and create the theme. + + Methods: + get_new_theme_name + create_new + """ + new_theme_name = self.get_new_theme_name('New Theme Name:') + if new_theme_name: + self.create_new(new_theme_name) + + def create_new(self, new_theme_name): + """Create a new custom theme with the given name. + + Create the new theme based on the previously active theme + with the current changes applied. Once it is saved, then + activate the new theme. + + Attributes accessed: + builtin_name + custom_name + + Attributes updated: + customlist + theme_source + + Method: + save_new + set_theme_type + """ + if self.theme_source.get(): + theme_type = 'default' + theme_name = self.builtin_name.get() + else: + theme_type = 'user' + theme_name = self.custom_name.get() + new_theme = idleConf.GetThemeDict(theme_type, theme_name) + # Apply any of the old theme's unsaved changes to the new theme. + if theme_name in changes['highlight']: + theme_changes = changes['highlight'][theme_name] + for element in theme_changes: + new_theme[element] = theme_changes[element] + # Save the new theme. + self.save_new(new_theme_name, new_theme) + # Change GUI over to the new theme. + custom_theme_list = idleConf.GetSectionList('user', 'highlight') + custom_theme_list.sort() + self.customlist.SetMenu(custom_theme_list, new_theme_name) + self.theme_source.set(0) + self.set_theme_type() + + def set_highlight_target(self): + """Set fg/bg toggle and color based on highlight tag target. + + Instance variables accessed: + highlight_target + + Attributes updated: + fg_on + bg_on + fg_bg_toggle + + Methods: + set_color_sample + + Called from: + var_changed_highlight_target + load_theme_cfg + """ + if self.highlight_target.get() == 'Cursor': # bg not possible + self.fg_on['state'] = DISABLED + self.bg_on['state'] = DISABLED + self.fg_bg_toggle.set(1) + else: # Both fg and bg can be set. + self.fg_on['state'] = NORMAL + self.bg_on['state'] = NORMAL + self.fg_bg_toggle.set(1) + self.set_color_sample() + + def set_color_sample_binding(self, *args): + """Change color sample based on foreground/background toggle. + + Methods: + set_color_sample + """ + self.set_color_sample() + + def set_color_sample(self): + """Set the color of the frame background to reflect the selected target. + + Instance variables accessed: + theme_elements + highlight_target + fg_bg_toggle + highlight_sample + + Attributes updated: + frame_color_set + """ + # Set the color sample area. + tag = self.theme_elements[self.highlight_target.get()][0] + plane = 'foreground' if self.fg_bg_toggle.get() else 'background' + color = self.highlight_sample.tag_cget(tag, plane) + self.frame_color_set['bg'] = color + + def paint_theme_sample(self): + """Apply the theme colors to each element tag in the sample text. + + Instance attributes accessed: + theme_elements + theme_source + builtin_name + custom_name + + Attributes updated: + highlight_sample: Set the tag elements to the theme. + + Methods: + set_color_sample + + Called from: + var_changed_builtin_name + var_changed_custom_name + load_theme_cfg + """ + if self.theme_source.get(): # Default theme + theme = self.builtin_name.get() + else: # User theme + theme = self.custom_name.get() + for element_title in self.theme_elements: + element = self.theme_elements[element_title][0] + colors = idleConf.GetHighlight(theme, element) + if element == 'cursor': # Cursor sample needs special painting. + colors['background'] = idleConf.GetHighlight( + theme, 'normal', fgBg='bg') + # Handle any unsaved changes to this theme. + if theme in changes['highlight']: + theme_dict = changes['highlight'][theme] + if element + '-foreground' in theme_dict: + colors['foreground'] = theme_dict[element + '-foreground'] + if element + '-background' in theme_dict: + colors['background'] = theme_dict[element + '-background'] + self.highlight_sample.tag_config(element, **colors) + self.set_color_sample() + + def save_new(self, theme_name, theme): + """Save a newly created theme to idleConf. + + theme_name - string, the name of the new theme + theme - dictionary containing the new theme + """ + if not idleConf.userCfg['highlight'].has_section(theme_name): + idleConf.userCfg['highlight'].add_section(theme_name) + for element in theme: + value = theme[element] + idleConf.userCfg['highlight'].SetOption(theme_name, element, value) + + def delete_custom(self): + """Handle event to delete custom theme. + + The current theme is deactivated and the default theme is + activated. The custom theme is permanently removed from + the config file. + + Attributes accessed: + custom_name + + Attributes updated: + custom_theme_on + customlist + theme_source + builtin_name + + Methods: + deactivate_current_config + save_all_changed_extensions + activate_config_changes + set_theme_type + """ + theme_name = self.custom_name.get() + delmsg = 'Are you sure you wish to delete the theme %r ?' + if not tkMessageBox.askyesno( + 'Delete Theme', delmsg % theme_name, parent=self): + return + cd.deactivate_current_config() + # Remove theme from changes, config, and file. + changes.delete_section('highlight', theme_name) + # Reload user theme list. + item_list = idleConf.GetSectionList('user', 'highlight') + item_list.sort() + if not item_list: + self.custom_theme_on['state'] = DISABLED + self.customlist.SetMenu(item_list, '- no custom themes -') + else: + self.customlist.SetMenu(item_list, item_list[0]) + # Revert to default theme. + self.theme_source.set(idleConf.defaultCfg['main'].Get('Theme', 'default')) + self.builtin_name.set(idleConf.defaultCfg['main'].Get('Theme', 'name')) + # User can't back out of these changes, they must be applied now. + changes.save_all() + cd.save_all_changed_extensions() + cd.activate_config_changes() + self.set_theme_type() + + class KeysPage(Frame): def __init__(self, master): diff --git a/Misc/NEWS.d/next/IDLE/2017-08-18-14-13-42.bpo-31206.F1-tKK.rst b/Misc/NEWS.d/next/IDLE/2017-08-18-14-13-42.bpo-31206.F1-tKK.rst new file mode 100644 index 00000000000..ba984065e88 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-08-18-14-13-42.bpo-31206.F1-tKK.rst @@ -0,0 +1,2 @@ +IDLE: Factor HighPage(Frame) class from ConfigDialog. Patch by Cheryl +Sabella.

1 0

bpo-31235: Fix ResourceWarning in test_logging (#3147)
by Victor Stinner 18 Aug '17

18 Aug '17

https://github.com/python/cpython/commit/a7719e27b3cad0f2b86cb932a76cbe55c5… commit: a7719e27b3cad0f2b86cb932a76cbe55c541b02e branch: master author: Victor Stinner <victor.stinner(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-08-19T00:34:00+02:00 summary: bpo-31235: Fix ResourceWarning in test_logging (#3147) files: M Lib/test/test_logging.py diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 226532a9792..f4aef9f8962 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -797,6 +797,8 @@ def stop(self, timeout=None): """ self.close() self._thread.join(timeout) + asyncore.close_all(map=self._map, ignore_all=True) + alive = self._thread.is_alive() self._thread = None if alive:

1 0