Mailman 3 August 2017 - Python-checkins

bpo-30830: test_logging uses threading_setup/cleanup (#3137)
by Victor Stinner 19 Aug '17

19 Aug '17

https://github.com/python/cpython/commit/6966960468327c958b03391f71f24986bd… commit: 6966960468327c958b03391f71f24986bd697307 branch: master author: Victor Stinner <victor.stinner(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-08-18T23:47:54+02:00 summary: bpo-30830: test_logging uses threading_setup/cleanup (#3137) * bpo-30830: test_logging uses threading_setup/cleanup Replace @support.reap_threads on some methods with support.threading_setup() in setUp() and support.threading_cleanup() in tearDown() in BaseTest. * bpo-30830: test_logging disables threaded socketserver tests Disable tests because of socketserver.ThreadingMixIn leaks threads, whereas leaking threads now makes a test to fail on buildbots. Disable tests until socketserver is fixed: bpo-31233. * Skip also setup_via_listener() files: M Lib/test/test_logging.py diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index a91cfd4ccd3..226532a9792 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -79,6 +79,8 @@ class BaseTest(unittest.TestCase): def setUp(self): """Setup the default logging stream to an internal StringIO instance, so that we can examine log output as we want.""" + self._threading_key = support.threading_setup() + logger_dict = logging.getLogger().manager.loggerDict logging._acquireLock() try: @@ -147,6 +149,9 @@ def tearDown(self): finally: logging._releaseLock() + self.doCleanups() + support.threading_cleanup(*self._threading_key) + def assert_log_lines(self, expected_values, stream=None, pat=None): """Match the collected log lines against the regular expression self.expected_log_pat, and compare the extracted group values to @@ -621,7 +626,6 @@ def test_path_objects(self): @unittest.skipIf(os.name == 'nt', 'WatchedFileHandler not appropriate for Windows.') @unittest.skipUnless(threading, 'Threading required for this test.') - @support.reap_threads def test_race(self): # Issue #14632 refers. def remove_loop(fname, tries): @@ -986,7 +990,6 @@ class TestUnixDatagramServer(TestUDPServer): class SMTPHandlerTest(BaseTest): TIMEOUT = 8.0 - @support.reap_threads def test_basic(self): sockmap = {} server = TestSMTPServer((support.HOST, 0), self.process_message, 0.001, @@ -1466,6 +1469,7 @@ def test_logger_disabling(self): self.assertFalse(logger.disabled) +(a)unittest.skipIf(True, "FIXME: bpo-30830") @unittest.skipUnless(threading, 'Threading required for this test.') class SocketHandlerTest(BaseTest): @@ -1565,6 +1569,7 @@ def _get_temp_domain_socket(): os.remove(fn) return fn +(a)unittest.skipIf(True, "FIXME: bpo-30830") @unittest.skipUnless(hasattr(socket, "AF_UNIX"), "Unix sockets required") @unittest.skipUnless(threading, 'Threading required for this test.') class UnixSocketHandlerTest(SocketHandlerTest): @@ -1583,6 +1588,7 @@ def tearDown(self): SocketHandlerTest.tearDown(self) support.unlink(self.address) +(a)unittest.skipIf(True, "FIXME: bpo-30830") @unittest.skipUnless(threading, 'Threading required for this test.') class DatagramHandlerTest(BaseTest): @@ -1649,6 +1655,7 @@ def test_output(self): self.handled.wait() self.assertEqual(self.log_output, "spam\neggs\n") +(a)unittest.skipIf(True, "FIXME: bpo-30830") @unittest.skipUnless(hasattr(socket, "AF_UNIX"), "Unix sockets required") @unittest.skipUnless(threading, 'Threading required for this test.') class UnixDatagramHandlerTest(DatagramHandlerTest): @@ -1736,6 +1743,7 @@ def test_output(self): self.handled.wait() self.assertEqual(self.log_output, b'<11>h\xc3\xa4m-sp\xc3\xa4m') +(a)unittest.skipIf(True, "FIXME: bpo-30830") @unittest.skipUnless(hasattr(socket, "AF_UNIX"), "Unix sockets required") @unittest.skipUnless(threading, 'Threading required for this test.') class UnixSysLogHandlerTest(SysLogHandlerTest): @@ -1754,6 +1762,7 @@ def tearDown(self): SysLogHandlerTest.tearDown(self) support.unlink(self.address) +(a)unittest.skipIf(True, "FIXME: bpo-30830") @unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 support required for this test.') @unittest.skipUnless(threading, 'Threading required for this test.') @@ -1795,7 +1804,6 @@ def handle_request(self, request): request.end_headers() self.handled.set() - @support.reap_threads def test_output(self): # The log message sent to the HTTPHandler is properly received. logger = logging.getLogger("http") @@ -2879,6 +2887,9 @@ def test_config14_ok(self): logging.warning('Exclamation') self.assertTrue(output.getvalue().endswith('Exclamation!\n')) + # listen() uses ConfigSocketReceiver which is based + # on socketserver.ThreadingTCPServer + @unittest.skipIf(True, "FIXME: bpo-30830") @unittest.skipUnless(threading, 'listen() needs threading to work') def setup_via_listener(self, text, verify=None): text = text.encode("utf-8") @@ -2911,7 +2922,6 @@ def setup_via_listener(self, text, verify=None): self.fail("join() timed out") @unittest.skipUnless(threading, 'Threading required for this test.') - @support.reap_threads def test_listen_config_10_ok(self): with support.captured_stdout() as output: self.setup_via_listener(json.dumps(self.config10)) @@ -2932,7 +2942,6 @@ def test_listen_config_10_ok(self): ], stream=output) @unittest.skipUnless(threading, 'Threading required for this test.') - @support.reap_threads def test_listen_config_1_ok(self): with support.captured_stdout() as output: self.setup_via_listener(textwrap.dedent(ConfigFileTest.config1)) @@ -2948,7 +2957,6 @@ def test_listen_config_1_ok(self): self.assert_log_lines([]) @unittest.skipUnless(threading, 'Threading required for this test.') - @support.reap_threads def test_listen_verify(self): def verify_fail(stuff): @@ -3232,7 +3240,6 @@ def setup_and_log(log_queue, ident): handler.close() @patch.object(logging.handlers.QueueListener, 'handle') - @support.reap_threads def test_handle_called_with_queue_queue(self, mock_handle): for i in range(self.repeat): log_queue = queue.Queue() @@ -3242,7 +3249,6 @@ def test_handle_called_with_queue_queue(self, mock_handle): @support.requires_multiprocessing_queue @patch.object(logging.handlers.QueueListener, 'handle') - @support.reap_threads def test_handle_called_with_mp_queue(self, mock_handle): for i in range(self.repeat): log_queue = multiprocessing.Queue() @@ -3261,7 +3267,6 @@ def get_all_from_queue(log_queue): return [] @support.requires_multiprocessing_queue - @support.reap_threads def test_no_messages_in_queue_after_stop(self): """ Five messages are logged then the QueueListener is stopped. This

1 0

bpo-30947: Update libexpat from 2.2.1 to 2.2.3 (#3106)
by Victor Stinner 19 Aug '17

19 Aug '17

https://github.com/python/cpython/commit/93d0cb58b4da2a88c56f472c6c19491cc7… commit: 93d0cb58b4da2a88c56f472c6c19491cc7a390df branch: master author: Victor Stinner <victor.stinner(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-08-18T23:43:54+02:00 summary: bpo-30947: Update libexpat from 2.2.1 to 2.2.3 (#3106) * bpo-30947: Update libexpat from 2.2.1 to 2.2.3 * Add NEWS entry * Add new loadlibrary.c * expat_external.h: restore include "pyexpatns.h" * PCbuild: add expat/loadlibrary.c * Define XML_POOR_ENTROPY to compile expat files: A Misc/NEWS.d/next/Security/2017-08-16-16-35-59.bpo-30947.iNMmm4.rst A Modules/expat/loadlibrary.c M Modules/expat/expat.h M Modules/expat/siphash.h M Modules/expat/xmlparse.c M Modules/expat/xmlrole.c M Modules/expat/xmltok.c M Modules/expat/xmltok_impl.c M PCbuild/_elementtree.vcxproj M PCbuild/_elementtree.vcxproj.filters M PCbuild/pyexpat.vcxproj M PCbuild/pyexpat.vcxproj.filters M setup.py diff --git a/Misc/NEWS.d/next/Security/2017-08-16-16-35-59.bpo-30947.iNMmm4.rst b/Misc/NEWS.d/next/Security/2017-08-16-16-35-59.bpo-30947.iNMmm4.rst new file mode 100644 index 00000000000..3caca9a79b4 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2017-08-16-16-35-59.bpo-30947.iNMmm4.rst @@ -0,0 +1,2 @@ +Upgrade libexpat embedded copy from version 2.2.1 to 2.2.3 to get security +fixes. diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index 28b0f954d41..7e5bbb7e393 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -24,7 +24,6 @@ extern "C" { struct XML_ParserStruct; typedef struct XML_ParserStruct *XML_Parser; -/* Should this be defined using stdbool.h when C99 is available? */ typedef unsigned char XML_Bool; #define XML_TRUE ((XML_Bool) 1) #define XML_FALSE ((XML_Bool) 0) @@ -1049,7 +1048,7 @@ XML_GetFeatureList(void); */ #define XML_MAJOR_VERSION 2 #define XML_MINOR_VERSION 2 -#define XML_MICRO_VERSION 1 +#define XML_MICRO_VERSION 3 #ifdef __cplusplus } diff --git a/Modules/expat/loadlibrary.c b/Modules/expat/loadlibrary.c new file mode 100644 index 00000000000..ffce868399b --- /dev/null +++ b/Modules/expat/loadlibrary.c @@ -0,0 +1,141 @@ +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 2016 - 2017, Steve Holme, <steve_holme(a)hotmail.com>. + * + * All rights reserved. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF + * THIRD PARTY RIGHTS. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH + * THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * Except as contained in this notice, the name of a copyright holder shall + * not be used in advertising or otherwise to promote the sale, use or other + * dealings in this Software without prior written authorization of the + * copyright holder. + * + ***************************************************************************/ + +#if defined(_WIN32) + +#include <windows.h> +#include <tchar.h> + + +HMODULE _Expat_LoadLibrary(LPCTSTR filename); + + +#if !defined(LOAD_WITH_ALTERED_SEARCH_PATH) +#define LOAD_WITH_ALTERED_SEARCH_PATH 0x00000008 +#endif + +#if !defined(LOAD_LIBRARY_SEARCH_SYSTEM32) +#define LOAD_LIBRARY_SEARCH_SYSTEM32 0x00000800 +#endif + +/* We use our own typedef here since some headers might lack these */ +typedef HMODULE (APIENTRY *LOADLIBRARYEX_FN)(LPCTSTR, HANDLE, DWORD); + +/* See function definitions in winbase.h */ +#ifdef UNICODE +# ifdef _WIN32_WCE +# define LOADLIBARYEX L"LoadLibraryExW" +# else +# define LOADLIBARYEX "LoadLibraryExW" +# endif +#else +# define LOADLIBARYEX "LoadLibraryExA" +#endif + + +/* + * _Expat_LoadLibrary() + * + * This is used to dynamically load DLLs using the most secure method available + * for the version of Windows that we are running on. + * + * Parameters: + * + * filename [in] - The filename or full path of the DLL to load. If only the + * filename is passed then the DLL will be loaded from the + * Windows system directory. + * + * Returns the handle of the module on success; otherwise NULL. + */ +HMODULE _Expat_LoadLibrary(LPCTSTR filename) +{ + HMODULE hModule = NULL; + LOADLIBRARYEX_FN pLoadLibraryEx = NULL; + + /* Get a handle to kernel32 so we can access it's functions at runtime */ + HMODULE hKernel32 = GetModuleHandle(TEXT("kernel32")); + if(!hKernel32) + return NULL; + + /* Attempt to find LoadLibraryEx() which is only available on Windows 2000 + and above */ + pLoadLibraryEx = (LOADLIBRARYEX_FN) GetProcAddress(hKernel32, LOADLIBARYEX); + + /* Detect if there's already a path in the filename and load the library if + there is. Note: Both back slashes and forward slashes have been supported + since the earlier days of DOS at an API level although they are not + supported by command prompt */ + if(_tcspbrk(filename, TEXT("\\/"))) { + /** !checksrc! disable BANNEDFUNC 1 **/ + hModule = pLoadLibraryEx ? + pLoadLibraryEx(filename, NULL, LOAD_WITH_ALTERED_SEARCH_PATH) : + LoadLibrary(filename); + } + /* Detect if KB2533623 is installed, as LOAD_LIBARY_SEARCH_SYSTEM32 is only + supported on Windows Vista, Windows Server 2008, Windows 7 and Windows + Server 2008 R2 with this patch or natively on Windows 8 and above */ + else if(pLoadLibraryEx && GetProcAddress(hKernel32, "AddDllDirectory")) { + /* Load the DLL from the Windows system directory */ + hModule = pLoadLibraryEx(filename, NULL, LOAD_LIBRARY_SEARCH_SYSTEM32); + } + else { + /* Attempt to get the Windows system path */ + UINT systemdirlen = GetSystemDirectory(NULL, 0); + if(systemdirlen) { + /* Allocate space for the full DLL path (Room for the null terminator + is included in systemdirlen) */ + size_t filenamelen = _tcslen(filename); + TCHAR *path = malloc(sizeof(TCHAR) * (systemdirlen + 1 + filenamelen)); + if(path && GetSystemDirectory(path, systemdirlen)) { + /* Calculate the full DLL path */ + _tcscpy(path + _tcslen(path), TEXT("\\")); + _tcscpy(path + _tcslen(path), filename); + + /* Load the DLL from the Windows system directory */ + /** !checksrc! disable BANNEDFUNC 1 **/ + hModule = pLoadLibraryEx ? + pLoadLibraryEx(path, NULL, LOAD_WITH_ALTERED_SEARCH_PATH) : + LoadLibrary(path); + + } + free(path); + } + } + + return hModule; +} + +#else /* defined(_WIN32) */ + +/* ISO C requires a translation unit to contain at least one declaration + [-Wempty-translation-unit] */ +typedef int _TRANSLATION_UNIT_LOAD_LIBRARY_C_NOT_EMTPY; + +#endif /* defined(_WIN32) */ diff --git a/Modules/expat/siphash.h b/Modules/expat/siphash.h index 453dd42f27e..581872df7b4 100644 --- a/Modules/expat/siphash.h +++ b/Modules/expat/siphash.h @@ -2,9 +2,8 @@ * siphash.h - SipHash-2-4 in a single header file * -------------------------------------------------------------------------- * Derived by William Ahern from the reference implementation[1] published[2] - * by Jean-Philippe Aumasson and Daniel J. Berstein. Licensed in kind. * by Jean-Philippe Aumasson and Daniel J. Berstein. - * Minimal changes by Sebastian Pipping on top, details below. + * Minimal changes by Sebastian Pipping and Victor Stinner on top, see below. * Licensed under the CC0 Public Domain Dedication license. * * 1. https://www.131002.net/siphash/siphash24.c @@ -12,14 +11,25 @@ * -------------------------------------------------------------------------- * HISTORY: * - * 2017-06-10 (Sebastian Pipping) + * 2017-07-25 (Vadim Zeitlin) + * - Fix use of SIPHASH_MAIN macro + * + * 2017-07-05 (Sebastian Pipping) + * - Use _SIP_ULL macro to not require a C++11 compiler if compiled as C++ + * - Add const qualifiers at two places + * - Ensure <=80 characters line length (assuming tab width 4) + * + * 2017-06-23 (Victor Stinner) + * - Address Win64 compile warnings + * + * 2017-06-18 (Sebastian Pipping) * - Clarify license note in the header * - Address C89 issues: * - Stop using inline keyword (and let compiler decide) - * - Turn integer suffix ULL to UL * - Replace _Bool by int * - Turn macro siphash24 into a function * - Address invalid conversion (void pointer) by explicit cast + * - Address lack of stdint.h for Visual Studio 2003 to 2008 * - Always expose sip24_valid (for self-tests) * * 2012-11-04 - Born. (William Ahern) @@ -76,7 +86,23 @@ #define SIPHASH_H #include <stddef.h> /* size_t */ -#include <stdint.h> /* uint64_t uint32_t uint8_t */ + +#if defined(_WIN32) && defined(_MSC_VER) && (_MSC_VER < 1600) + /* For vs2003/7.1 up to vs2008/9.0; _MSC_VER 1600 is vs2010/10.0 */ + typedef unsigned __int8 uint8_t; + typedef unsigned __int32 uint32_t; + typedef unsigned __int64 uint64_t; +#else + #include <stdint.h> /* uint64_t uint32_t uint8_t */ +#endif + + +/* + * Workaround to not require a C++11 compiler for using ULL suffix + * if this code is included and compiled as C++; related GCC warning is: + * warning: use of C++11 long long integer constant [-Wlong-long] + */ +#define _SIP_ULL(high, low) (((uint64_t)high << 32) | low) #define SIP_ROTL(x, b) (uint64_t)(((x) << (b)) | ( (x) >> (64 - (b)))) @@ -158,11 +184,12 @@ static void sip_round(struct siphash *H, const int rounds) { } /* sip_round() */ -static struct siphash *sip24_init(struct siphash *H, const struct sipkey *key) { - H->v0 = 0x736f6d6570736575UL ^ key->k[0]; - H->v1 = 0x646f72616e646f6dUL ^ key->k[1]; - H->v2 = 0x6c7967656e657261UL ^ key->k[0]; - H->v3 = 0x7465646279746573UL ^ key->k[1]; +static struct siphash *sip24_init(struct siphash *H, + const struct sipkey *key) { + H->v0 = _SIP_ULL(0x736f6d65U, 0x70736575U) ^ key->k[0]; + H->v1 = _SIP_ULL(0x646f7261U, 0x6e646f6dU) ^ key->k[1]; + H->v2 = _SIP_ULL(0x6c796765U, 0x6e657261U) ^ key->k[0]; + H->v3 = _SIP_ULL(0x74656462U, 0x79746573U) ^ key->k[1]; H->p = H->buf; H->c = 0; @@ -173,7 +200,8 @@ static struct siphash *sip24_init(struct siphash *H, const struct sipkey *key) { #define sip_endof(a) (&(a)[sizeof (a) / sizeof *(a)]) -static struct siphash *sip24_update(struct siphash *H, const void *src, size_t len) { +static struct siphash *sip24_update(struct siphash *H, const void *src, + size_t len) { const unsigned char *p = (const unsigned char *)src, *pe = p + len; uint64_t m; @@ -198,7 +226,7 @@ static struct siphash *sip24_update(struct siphash *H, const void *src, size_t l static uint64_t sip24_final(struct siphash *H) { - char left = (char)(H->p - H->buf); + const char left = (char)(H->p - H->buf); uint64_t b = (H->c + left) << 56; switch (left) { @@ -222,7 +250,8 @@ static uint64_t sip24_final(struct siphash *H) { } /* sip24_final() */ -static uint64_t siphash24(const void *src, size_t len, const struct sipkey *key) { +static uint64_t siphash24(const void *src, size_t len, + const struct sipkey *key) { struct siphash state = SIPHASH_INITIALIZER; return sip24_final(sip24_update(sip24_init(&state, key), src, len)); } /* siphash24() */ @@ -310,7 +339,8 @@ static int sip24_valid(void) { struct sipkey k; size_t i; - sip_tokey(&k, "\000\001\002\003\004\005\006\007\010\011\012\013\014\015\016\017"); + sip_tokey(&k, "\000\001\002\003\004\005\006\007\010\011" + "\012\013\014\015\016\017"); for (i = 0; i < sizeof in; ++i) { in[i] = (unsigned char)i; @@ -323,12 +353,12 @@ static int sip24_valid(void) { } /* sip24_valid() */ -#if SIPHASH_MAIN +#ifdef SIPHASH_MAIN #include <stdio.h> int main(void) { - int ok = sip24_valid(); + const int ok = sip24_valid(); if (ok) puts("OK"); diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index daec151e232..b703e61a040 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -1,10 +1,12 @@ /* Copyright (c) 1998, 1999, 2000 Thai Open Source Software Center Ltd See the file COPYING for copying permission. - 77fea421d361dca90041d0040ecf1dca651167fadf2af79e990e35168d70d933 (2.2.1+) + 101bfd65d1ff3d1511cf6671e6aae65f82cd97df6f4da137d46d510731830ad9 (2.2.3+) */ -#define _GNU_SOURCE 1 /* syscall prototype */ +#if !defined(_GNU_SOURCE) +# define _GNU_SOURCE 1 /* syscall prototype */ +#endif #include <stddef.h> #include <string.h> /* memset(), memcpy() */ @@ -19,6 +21,8 @@ #include <sys/time.h> /* gettimeofday() */ #include <sys/types.h> /* getpid() */ #include <unistd.h> /* getpid() */ +#include <fcntl.h> /* O_RDONLY */ +#include <errno.h> #endif #define XML_BUILDING_EXPAT 1 @@ -33,6 +37,57 @@ #include "expat.h" #include "siphash.h" +#if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) +# if defined(HAVE_GETRANDOM) +# include <sys/random.h> /* getrandom */ +# else +# include <unistd.h> /* syscall */ +# include <sys/syscall.h> /* SYS_getrandom */ +# endif +# if ! defined(GRND_NONBLOCK) +# define GRND_NONBLOCK 0x0001 +# endif /* defined(GRND_NONBLOCK) */ +#endif /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */ + +#if defined(HAVE_LIBBSD) \ + && (defined(HAVE_ARC4RANDOM_BUF) || defined(HAVE_ARC4RANDOM)) +# include <bsd/stdlib.h> +#endif + +#if defined(_WIN32) && !defined(LOAD_LIBRARY_SEARCH_SYSTEM32) +# define LOAD_LIBRARY_SEARCH_SYSTEM32 0x00000800 +#endif + +#if !defined(HAVE_GETRANDOM) && !defined(HAVE_SYSCALL_GETRANDOM) \ + && !defined(HAVE_ARC4RANDOM_BUF) && !defined(HAVE_ARC4RANDOM) \ + && !defined(XML_DEV_URANDOM) \ + && !defined(_WIN32) \ + && !defined(XML_POOR_ENTROPY) +# error \ + You do not have support for any sources of high quality entropy \ + enabled. For end user security, that is probably not what you want. \ + \ + Your options include: \ + * Linux + glibc >=2.25 (getrandom): HAVE_GETRANDOM, \ + * Linux + glibc <2.25 (syscall SYS_getrandom): HAVE_SYSCALL_GETRANDOM, \ + * BSD / macOS >=10.7 (arc4random_buf): HAVE_ARC4RANDOM_BUF, \ + * BSD / macOS <10.7 (arc4random): HAVE_ARC4RANDOM, \ + * libbsd (arc4random_buf): HAVE_ARC4RANDOM_BUF + HAVE_LIBBSD, \ + * libbsd (arc4random): HAVE_ARC4RANDOM + HAVE_LIBBSD, \ + * Linux / BSD / macOS (/dev/urandom): XML_DEV_URANDOM \ + * Windows (RtlGenRandom): _WIN32. \ + \ + If insist on not using any of these, bypass this error by defining \ + XML_POOR_ENTROPY; you have been warned. \ + \ + For CMake, one way to pass the define is: \ + cmake -DCMAKE_C_FLAGS="-pipe -O2 -DHAVE_SYSCALL_GETRANDOM" . \ + \ + If you have reasons to patch this detection code away or need changes \ + to the build system, please open a bug. Thank you! +#endif + + #ifdef XML_UNICODE #define XML_ENCODE_MAX XML_UTF16_ENCODE_MAX #define XmlConvert XmlUtf16Convert @@ -436,6 +491,9 @@ static ELEMENT_TYPE * getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr, const char *end); +static XML_Char *copyString(const XML_Char *s, + const XML_Memory_Handling_Suite *memsuite); + static unsigned long generate_hash_secret_salt(XML_Parser parser); static XML_Bool startParsing(XML_Parser parser); @@ -696,21 +754,13 @@ static const XML_Char implicitContext[] = { #if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) -# include <errno.h> - -# if defined(HAVE_GETRANDOM) -# include <sys/random.h> /* getrandom */ -# else -# include <unistd.h> /* syscall */ -# include <sys/syscall.h> /* SYS_getrandom */ -# endif /* Obtain entropy on Linux 3.17+ */ static int -writeRandomBytes_getrandom(void * target, size_t count) { +writeRandomBytes_getrandom_nonblock(void * target, size_t count) { int success = 0; /* full count bytes written? */ size_t bytesWrittenTotal = 0; - const unsigned int getrandomFlags = 0; + const unsigned int getrandomFlags = GRND_NONBLOCK; do { void * const currentTarget = (void*)((char*)target + bytesWrittenTotal); @@ -728,7 +778,7 @@ writeRandomBytes_getrandom(void * target, size_t count) { if (bytesWrittenTotal >= count) success = 1; } - } while (! success && (errno == EINTR || errno == EAGAIN)); + } while (! success && (errno == EINTR)); return success; } @@ -736,12 +786,67 @@ writeRandomBytes_getrandom(void * target, size_t count) { #endif /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */ +#if ! defined(_WIN32) && defined(XML_DEV_URANDOM) + +/* Extract entropy from /dev/urandom */ +static int +writeRandomBytes_dev_urandom(void * target, size_t count) { + int success = 0; /* full count bytes written? */ + size_t bytesWrittenTotal = 0; + + const int fd = open("/dev/urandom", O_RDONLY); + if (fd < 0) { + return 0; + } + + do { + void * const currentTarget = (void*)((char*)target + bytesWrittenTotal); + const size_t bytesToWrite = count - bytesWrittenTotal; + + const ssize_t bytesWrittenMore = read(fd, currentTarget, bytesToWrite); + + if (bytesWrittenMore > 0) { + bytesWrittenTotal += bytesWrittenMore; + if (bytesWrittenTotal >= count) + success = 1; + } + } while (! success && (errno == EINTR)); + + close(fd); + return success; +} + +#endif /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */ + + +#if defined(HAVE_ARC4RANDOM) + +static void +writeRandomBytes_arc4random(void * target, size_t count) { + size_t bytesWrittenTotal = 0; + + while (bytesWrittenTotal < count) { + const uint32_t random32 = arc4random(); + size_t i = 0; + + for (; (i < sizeof(random32)) && (bytesWrittenTotal < count); + i++, bytesWrittenTotal++) { + const uint8_t random8 = (uint8_t)(random32 >> (i * 8)); + ((uint8_t *)target)[bytesWrittenTotal] = random8; + } + } +} + +#endif /* defined(HAVE_ARC4RANDOM) */ + + #ifdef _WIN32 typedef BOOLEAN (APIENTRY *RTLGENRANDOM_FUNC)(PVOID, ULONG); +HMODULE _Expat_LoadLibrary(LPCTSTR filename); /* see loadlibrary.c */ /* Obtain entropy on Windows XP / Windows Server 2003 and later. - * Hint on RtlGenRandom and the following article from libsodioum. + * Hint on RtlGenRandom and the following article from libsodium. * * Michael Howard: Cryptographically Secure Random number on Windows without using CryptoAPI * https://blogs.msdn.microsoft.com/michael_howard/2005/01/14/cryptographicall… @@ -749,7 +854,7 @@ typedef BOOLEAN (APIENTRY *RTLGENRANDOM_FUNC)(PVOID, ULONG); static int writeRandomBytes_RtlGenRandom(void * target, size_t count) { int success = 0; /* full count bytes written? */ - const HMODULE advapi32 = LoadLibrary("ADVAPI32.DLL"); + const HMODULE advapi32 = _Expat_LoadLibrary(TEXT("ADVAPI32.DLL")); if (advapi32) { const RTLGENRANDOM_FUNC RtlGenRandom @@ -768,6 +873,8 @@ writeRandomBytes_RtlGenRandom(void * target, size_t count) { #endif /* _WIN32 */ +#if ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) + static unsigned long gather_time_entropy(void) { @@ -780,16 +887,20 @@ gather_time_entropy(void) int gettimeofday_res; gettimeofday_res = gettimeofday(&tv, NULL); + +#if defined(NDEBUG) + (void)gettimeofday_res; +#else assert (gettimeofday_res == 0); +#endif /* defined(NDEBUG) */ /* Microseconds time is <20 bits entropy */ return tv.tv_usec; #endif } -#if defined(HAVE_ARC4RANDOM_BUF) && defined(HAVE_LIBBSD) -# include <bsd/stdlib.h> -#endif +#endif /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */ + static unsigned long ENTROPY_DEBUG(const char * label, unsigned long entropy) { @@ -808,10 +919,12 @@ generate_hash_secret_salt(XML_Parser parser) { unsigned long entropy; (void)parser; -#if defined(HAVE_ARC4RANDOM_BUF) || defined(__CloudABI__) - (void)gather_time_entropy; +#if defined(HAVE_ARC4RANDOM_BUF) arc4random_buf(&entropy, sizeof(entropy)); return ENTROPY_DEBUG("arc4random_buf", entropy); +#elif defined(HAVE_ARC4RANDOM) + writeRandomBytes_arc4random((void *)&entropy, sizeof(entropy)); + return ENTROPY_DEBUG("arc4random", entropy); #else /* Try high quality providers first .. */ #ifdef _WIN32 @@ -819,10 +932,15 @@ generate_hash_secret_salt(XML_Parser parser) return ENTROPY_DEBUG("RtlGenRandom", entropy); } #elif defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) - if (writeRandomBytes_getrandom((void *)&entropy, sizeof(entropy))) { + if (writeRandomBytes_getrandom_nonblock((void *)&entropy, sizeof(entropy))) { return ENTROPY_DEBUG("getrandom", entropy); } #endif +#if ! defined(_WIN32) && defined(XML_DEV_URANDOM) + if (writeRandomBytes_dev_urandom((void *)&entropy, sizeof(entropy))) { + return ENTROPY_DEBUG("/dev/urandom", entropy); + } +#endif /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */ /* .. and self-made low quality for backup: */ /* Process ID is 0 bits entropy if attacker has local access */ @@ -833,7 +951,7 @@ generate_hash_secret_salt(XML_Parser parser) return ENTROPY_DEBUG("fallback(4)", entropy * 2147483647); } else { return ENTROPY_DEBUG("fallback(8)", - entropy * (unsigned long)2305843009213693951); + entropy * (unsigned long)2305843009213693951ULL); } #endif } @@ -962,6 +1080,8 @@ parserCreate(const XML_Char *encodingName, nsAttsVersion = 0; nsAttsPower = 0; + protocolEncodingName = NULL; + poolInit(&tempPool, &(parser->m_mem)); poolInit(&temp2Pool, &(parser->m_mem)); parserInit(parser, encodingName); @@ -988,9 +1108,9 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { processor = prologInitProcessor; XmlPrologStateInit(&prologState); - protocolEncodingName = (encodingName != NULL - ? poolCopyString(&tempPool, encodingName) - : NULL); + if (encodingName != NULL) { + protocolEncodingName = copyString(encodingName, &(parser->m_mem)); + } curBase = NULL; XmlInitEncoding(&initEncoding, &encoding, 0); userData = NULL; @@ -1103,6 +1223,8 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) unknownEncodingRelease(unknownEncodingData); poolClear(&tempPool); poolClear(&temp2Pool); + FREE((void *)protocolEncodingName); + protocolEncodingName = NULL; parserInit(parser, encodingName); dtdReset(_dtd, &parser->m_mem); return XML_TRUE; @@ -1119,10 +1241,16 @@ XML_SetEncoding(XML_Parser parser, const XML_Char *encodingName) */ if (ps_parsing == XML_PARSING || ps_parsing == XML_SUSPENDED) return XML_STATUS_ERROR; + + /* Get rid of any previous encoding name */ + FREE((void *)protocolEncodingName); + if (encodingName == NULL) + /* No new encoding name */ protocolEncodingName = NULL; else { - protocolEncodingName = poolCopyString(&tempPool, encodingName); + /* Copy the new encoding name into allocated memory */ + protocolEncodingName = copyString(encodingName, &(parser->m_mem)); if (!protocolEncodingName) return XML_STATUS_ERROR; } @@ -1357,6 +1485,7 @@ XML_ParserFree(XML_Parser parser) destroyBindings(inheritedBindings, parser); poolDestroy(&tempPool); poolDestroy(&temp2Pool); + FREE((void *)protocolEncodingName); #ifdef XML_DTD /* external parameter entity parsers share the DTD structure parser->m_dtd with the root parser, so we must not destroy it @@ -1748,7 +1877,8 @@ enum XML_Status XMLCALL XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { if ((parser == NULL) || (len < 0) || ((s == NULL) && (len != 0))) { - errorCode = XML_ERROR_INVALID_ARGUMENT; + if (parser != NULL) + parser->m_errorCode = XML_ERROR_INVALID_ARGUMENT; return XML_STATUS_ERROR; } switch (ps_parsing) { @@ -1783,9 +1913,22 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) if (errorCode == XML_ERROR_NONE) { switch (ps_parsing) { case XML_SUSPENDED: + /* It is hard to be certain, but it seems that this case + * cannot occur. This code is cleaning up a previous parse + * with no new data (since len == 0). Changing the parsing + * state requires getting to execute a handler function, and + * there doesn't seem to be an opportunity for that while in + * this circumstance. + * + * Given the uncertainty, we retain the code but exclude it + * from coverage tests. + * + * LCOV_EXCL_START + */ XmlUpdatePosition(encoding, positionPtr, bufferPtr, &position); positionPtr = bufferPtr; return XML_STATUS_SUSPENDED; + /* LCOV_EXCL_STOP */ case XML_INITIALIZED: case XML_PARSING: ps_parsing = XML_FINISHED; @@ -2974,9 +3117,17 @@ doContent(XML_Parser parser, return XML_ERROR_NO_MEMORY; break; default: + /* All of the tokens produced by XmlContentTok() have their own + * explicit cases, so this default is not strictly necessary. + * However it is a useful safety net, so we retain the code and + * simply exclude it from the coverage tests. + * + * LCOV_EXCL_START + */ if (defaultHandler) reportDefault(parser, enc, s, next); break; + /* LCOV_EXCL_STOP */ } *eventPP = s = next; switch (ps_parsing) { @@ -3067,13 +3218,17 @@ storeAtts(XML_Parser parser, const ENCODING *enc, #endif attsSize = n + nDefaultAtts + INIT_ATTS_SIZE; temp = (ATTRIBUTE *)REALLOC((void *)atts, attsSize * sizeof(ATTRIBUTE)); - if (temp == NULL) + if (temp == NULL) { + attsSize = oldAttsSize; return XML_ERROR_NO_MEMORY; + } atts = temp; #ifdef XML_ATTR_INFO temp2 = (XML_AttrInfo *)REALLOC((void *)attInfo, attsSize * sizeof(XML_AttrInfo)); - if (temp2 == NULL) + if (temp2 == NULL) { + attsSize = oldAttsSize; return XML_ERROR_NO_MEMORY; + } attInfo = temp2; #endif if (n > oldAttsSize) @@ -3210,6 +3365,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, int j; /* hash table index */ unsigned long version = nsAttsVersion; int nsAttsSize = (int)1 << nsAttsPower; + unsigned char oldNsAttsPower = nsAttsPower; /* size of hash table must be at least 2 * (# of prefixed attributes) */ if ((nPrefixes << 1) >> nsAttsPower) { /* true for nsAttsPower = 0 */ NS_ATT *temp; @@ -3219,8 +3375,11 @@ storeAtts(XML_Parser parser, const ENCODING *enc, nsAttsPower = 3; nsAttsSize = (int)1 << nsAttsPower; temp = (NS_ATT *)REALLOC(nsAtts, nsAttsSize * sizeof(NS_ATT)); - if (!temp) + if (!temp) { + /* Restore actual size of memory in nsAtts */ + nsAttsPower = oldNsAttsPower; return XML_ERROR_NO_MEMORY; + } nsAtts = temp; version = 0; /* force re-initialization of nsAtts hash table */ } @@ -3247,8 +3406,23 @@ storeAtts(XML_Parser parser, const ENCODING *enc, ((XML_Char *)s)[-1] = 0; /* clear flag */ id = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, s, 0); - if (!id || !id->prefix) - return XML_ERROR_NO_MEMORY; + if (!id || !id->prefix) { + /* This code is walking through the appAtts array, dealing + * with (in this case) a prefixed attribute name. To be in + * the array, the attribute must have already been bound, so + * has to have passed through the hash table lookup once + * already. That implies that an entry for it already + * exists, so the lookup above will return a pointer to + * already allocated memory. There is no opportunaity for + * the allocator to fail, so the condition above cannot be + * fulfilled. + * + * Since it is difficult to be certain that the above + * analysis is complete, we retain the test and merely + * remove the code from coverage tests. + */ + return XML_ERROR_NO_MEMORY; /* LCOV_EXCL_LINE */ + } b = id->prefix->binding; if (!b) return XML_ERROR_UNBOUND_PREFIX; @@ -3625,8 +3799,16 @@ doCdataSection(XML_Parser parser, } return XML_ERROR_UNCLOSED_CDATA_SECTION; default: + /* Every token returned by XmlCdataSectionTok() has its own + * explicit case, so this default case will never be executed. + * We retain it as a safety net and exclude it from the coverage + * statistics. + * + * LCOV_EXCL_START + */ *eventPP = next; return XML_ERROR_UNEXPECTED_STATE; + /* LCOV_EXCL_STOP */ } *eventPP = s = next; @@ -3686,8 +3868,20 @@ doIgnoreSection(XML_Parser parser, eventEndPP = &eventEndPtr; } else { + /* It's not entirely clear, but it seems the following two lines + * of code cannot be executed. The only occasions on which 'enc' + * is not 'parser->m_encoding' are when this function is called + * from the internal entity processing, and IGNORE sections are an + * error in internal entities. + * + * Since it really isn't clear that this is true, we keep the code + * and just remove it from our coverage tests. + * + * LCOV_EXCL_START + */ eventPP = &(openInternalEntities->internalEventPtr); eventEndPP = &(openInternalEntities->internalEventEndPtr); + /* LCOV_EXCL_STOP */ } *eventPP = s; *startPtr = NULL; @@ -3720,8 +3914,16 @@ doIgnoreSection(XML_Parser parser, } return XML_ERROR_SYNTAX; /* XML_ERROR_UNCLOSED_IGNORE_SECTION */ default: + /* All of the tokens that XmlIgnoreSectionTok() returns have + * explicit cases to handle them, so this default case is never + * executed. We keep it as a safety net anyway, and remove it + * from our test coverage statistics. + * + * LCOV_EXCL_START + */ *eventPP = next; return XML_ERROR_UNEXPECTED_STATE; + /* LCOV_EXCL_STOP */ } /* not reached */ } @@ -3734,6 +3936,7 @@ initializeEncoding(XML_Parser parser) const char *s; #ifdef XML_UNICODE char encodingBuf[128]; + /* See comments abount `protoclEncodingName` in parserInit() */ if (!protocolEncodingName) s = NULL; else { @@ -3817,7 +4020,14 @@ processXmlDecl(XML_Parser parser, int isGeneralTextEntity, reportDefault(parser, encoding, s, next); if (protocolEncodingName == NULL) { if (newEncoding) { - if (newEncoding->minBytesPerChar != encoding->minBytesPerChar) { + /* Check that the specified encoding does not conflict with what + * the parser has already deduced. Do we have the same number + * of bytes in the smallest representation of a character? If + * this is UTF-16, is it the same endianness? + */ + if (newEncoding->minBytesPerChar != encoding->minBytesPerChar + || (newEncoding->minBytesPerChar == 2 && + newEncoding != encoding)) { eventPtr = encodingName; return XML_ERROR_INCORRECT_ENCODING; } @@ -3962,15 +4172,14 @@ entityValueInitProcessor(XML_Parser parser, result = processXmlDecl(parser, 0, start, next); if (result != XML_ERROR_NONE) return result; - switch (ps_parsing) { - case XML_SUSPENDED: - *nextPtr = next; - return XML_ERROR_NONE; - case XML_FINISHED: + /* At this point, ps_parsing cannot be XML_SUSPENDED. For that + * to happen, a parameter entity parsing handler must have + * attempted to suspend the parser, which fails and raises an + * error. The parser can be aborted, but can't be suspended. + */ + if (ps_parsing == XML_FINISHED) return XML_ERROR_ABORTED; - default: - *nextPtr = next; - } + *nextPtr = next; /* stop scanning for text declaration - we found one */ processor = entityValueProcessor; return entityValueProcessor(parser, next, end, nextPtr); @@ -4293,8 +4502,14 @@ doProlog(XML_Parser parser, &dtd->paramEntities, externalSubsetName, sizeof(ENTITY)); - if (!entity) - return XML_ERROR_NO_MEMORY; + if (!entity) { + /* The external subset name "#" will have already been + * inserted into the hash table at the start of the + * external entity parsing, so no allocation will happen + * and lookup() cannot fail. + */ + return XML_ERROR_NO_MEMORY; /* LCOV_EXCL_LINE */ + } if (useForeignDTD) entity->base = curBase; dtd->paramEntityRead = XML_FALSE; @@ -4773,8 +4988,10 @@ doProlog(XML_Parser parser, if (prologState.level >= groupSize) { if (groupSize) { char *temp = (char *)REALLOC(groupConnector, groupSize *= 2); - if (temp == NULL) + if (temp == NULL) { + groupSize /= 2; return XML_ERROR_NO_MEMORY; + } groupConnector = temp; if (dtd->scaffIndex) { int *temp = (int *)REALLOC(dtd->scaffIndex, @@ -4786,8 +5003,10 @@ doProlog(XML_Parser parser, } else { groupConnector = (char *)MALLOC(groupSize = 32); - if (!groupConnector) + if (!groupConnector) { + groupSize = 0; return XML_ERROR_NO_MEMORY; + } } } groupConnector[prologState.level] = 0; @@ -4850,8 +5069,29 @@ doProlog(XML_Parser parser, : !dtd->hasParamEntityRefs)) { if (!entity) return XML_ERROR_UNDEFINED_ENTITY; - else if (!entity->is_internal) - return XML_ERROR_ENTITY_DECLARED_IN_PE; + else if (!entity->is_internal) { + /* It's hard to exhaustively search the code to be sure, + * but there doesn't seem to be a way of executing the + * following line. There are two cases: + * + * If 'standalone' is false, the DTD must have no + * parameter entities or we wouldn't have passed the outer + * 'if' statement. That measn the only entity in the hash + * table is the external subset name "#" which cannot be + * given as a parameter entity name in XML syntax, so the + * lookup must have returned NULL and we don't even reach + * the test for an internal entity. + * + * If 'standalone' is true, it does not seem to be + * possible to create entities taking this code path that + * are not internal entities, so fail the test above. + * + * Because this analysis is very uncertain, the code is + * being left in place and merely removed from the + * coverage test statistics. + */ + return XML_ERROR_ENTITY_DECLARED_IN_PE; /* LCOV_EXCL_LINE */ + } } else if (!entity) { dtd->keepProcessing = dtd->standalone; @@ -5323,11 +5563,15 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, && (poolLength(pool) == 0 || poolLastChar(pool) == 0x20)) break; n = XmlEncode(n, (ICHAR *)buf); - if (!n) { - if (enc == encoding) - eventPtr = ptr; - return XML_ERROR_BAD_CHAR_REF; - } + /* The XmlEncode() functions can never return 0 here. That + * error return happens if the code point passed in is either + * negative or greater than or equal to 0x110000. The + * XmlCharRefNumber() functions will all return a number + * strictly less than 0x110000 or a negative value if an error + * occurred. The negative value is intercepted above, so + * XmlEncode() is never passed a value it might return an + * error for. + */ for (i = 0; i < n; i++) { if (!poolAppendChar(pool, buf[i])) return XML_ERROR_NO_MEMORY; @@ -5401,8 +5645,26 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, break; } if (entity->open) { - if (enc == encoding) - eventPtr = ptr; + if (enc == encoding) { + /* It does not appear that this line can be executed. + * + * The "if (entity->open)" check catches recursive entity + * definitions. In order to be called with an open + * entity, it must have gone through this code before and + * been through the recursive call to + * appendAttributeValue() some lines below. That call + * sets the local encoding ("enc") to the parser's + * internal encoding (internal_utf8 or internal_utf16), + * which can never be the same as the principle encoding. + * It doesn't appear there is another code path that gets + * here with entity->open being TRUE. + * + * Since it is not certain that this logic is watertight, + * we keep the line and merely exclude it from coverage + * tests. + */ + eventPtr = ptr; /* LCOV_EXCL_LINE */ + } return XML_ERROR_RECURSIVE_ENTITY_REF; } if (entity->notation) { @@ -5429,9 +5691,21 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, } break; default: + /* The only token returned by XmlAttributeValueTok() that does + * not have an explicit case here is XML_TOK_PARTIAL_CHAR. + * Getting that would require an entity name to contain an + * incomplete XML character (e.g. \xE2\x82); however previous + * tokenisers will have already recognised and rejected such + * names before XmlAttributeValueTok() gets a look-in. This + * default case should be retained as a safety net, but the code + * excluded from coverage tests. + * + * LCOV_EXCL_START + */ if (enc == encoding) eventPtr = ptr; return XML_ERROR_UNEXPECTED_STATE; + /* LCOV_EXCL_STOP */ } ptr = next; } @@ -5564,12 +5838,15 @@ storeEntityValue(XML_Parser parser, goto endEntityValue; } n = XmlEncode(n, (ICHAR *)buf); - if (!n) { - if (enc == encoding) - eventPtr = entityTextPtr; - result = XML_ERROR_BAD_CHAR_REF; - goto endEntityValue; - } + /* The XmlEncode() functions can never return 0 here. That + * error return happens if the code point passed in is either + * negative or greater than or equal to 0x110000. The + * XmlCharRefNumber() functions will all return a number + * strictly less than 0x110000 or a negative value if an error + * occurred. The negative value is intercepted above, so + * XmlEncode() is never passed a value it might return an + * error for. + */ for (i = 0; i < n; i++) { if (pool->end == pool->ptr && !poolGrow(pool)) { result = XML_ERROR_NO_MEMORY; @@ -5590,10 +5867,18 @@ storeEntityValue(XML_Parser parser, result = XML_ERROR_INVALID_TOKEN; goto endEntityValue; default: + /* This default case should be unnecessary -- all the tokens + * that XmlEntityValueTok() can return have their own explicit + * cases -- but should be retained for safety. We do however + * exclude it from the coverage statistics. + * + * LCOV_EXCL_START + */ if (enc == encoding) eventPtr = entityTextPtr; result = XML_ERROR_UNEXPECTED_STATE; goto endEntityValue; + /* LCOV_EXCL_STOP */ } entityTextPtr = next; } @@ -5691,8 +5976,25 @@ reportDefault(XML_Parser parser, const ENCODING *enc, eventEndPP = &eventEndPtr; } else { + /* To get here, two things must be true; the parser must be + * using a character encoding that is not the same as the + * encoding passed in, and the encoding passed in must need + * conversion to the internal format (UTF-8 unless XML_UNICODE + * is defined). The only occasions on which the encoding passed + * in is not the same as the parser's encoding are when it is + * the internal encoding (e.g. a previously defined parameter + * entity, already converted to internal format). This by + * definition doesn't need conversion, so the whole branch never + * gets executed. + * + * For safety's sake we don't delete these lines and merely + * exclude them from coverage statistics. + * + * LCOV_EXCL_START + */ eventPP = &(openInternalEntities->internalEventPtr); eventEndPP = &(openInternalEntities->internalEventEndPtr); + /* LCOV_EXCL_STOP */ } do { ICHAR *dataPtr = (ICHAR *)dataBuf; @@ -5861,9 +6163,30 @@ getContext(XML_Parser parser) len = dtd->defaultPrefix.binding->uriLen; if (namespaceSeparator) len--; - for (i = 0; i < len; i++) - if (!poolAppendChar(&tempPool, dtd->defaultPrefix.binding->uri[i])) - return NULL; + for (i = 0; i < len; i++) { + if (!poolAppendChar(&tempPool, dtd->defaultPrefix.binding->uri[i])) { + /* Because of memory caching, I don't believe this line can be + * executed. + * + * This is part of a loop copying the default prefix binding + * URI into the parser's temporary string pool. Previously, + * that URI was copied into the same string pool, with a + * terminating NUL character, as part of setContext(). When + * the pool was cleared, that leaves a block definitely big + * enough to hold the URI on the free block list of the pool. + * The URI copy in getContext() therefore cannot run out of + * memory. + * + * If the pool is used between the setContext() and + * getContext() calls, the worst it can do is leave a bigger + * block on the front of the free list. Given that this is + * all somewhat inobvious and program logic can be changed, we + * don't delete the line but we do exclude it from the test + * coverage statistics. + */ + return NULL; /* LCOV_EXCL_LINE */ + } + } needSep = XML_TRUE; } @@ -5875,8 +6198,15 @@ getContext(XML_Parser parser) PREFIX *prefix = (PREFIX *)hashTableIterNext(&iter); if (!prefix) break; - if (!prefix->binding) - continue; + if (!prefix->binding) { + /* This test appears to be (justifiable) paranoia. There does + * not seem to be a way of injecting a prefix without a binding + * that doesn't get errored long before this function is called. + * The test should remain for safety's sake, so we instead + * exclude the following line from the coverage statistics. + */ + continue; /* LCOV_EXCL_LINE */ + } if (needSep && !poolAppendChar(&tempPool, CONTEXT_SEP)) return NULL; for (s = prefix->name; *s; s++) @@ -6547,8 +6877,20 @@ poolCopyString(STRING_POOL *pool, const XML_Char *s) static const XML_Char * poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n) { - if (!pool->ptr && !poolGrow(pool)) - return NULL; + if (!pool->ptr && !poolGrow(pool)) { + /* The following line is unreachable given the current usage of + * poolCopyStringN(). Currently it is called from exactly one + * place to copy the text of a simple general entity. By that + * point, the name of the entity is already stored in the pool, so + * pool->ptr cannot be NULL. + * + * If poolCopyStringN() is used elsewhere as it well might be, + * this line may well become executable again. Regardless, this + * sort of check shouldn't be removed lightly, so we just exclude + * it from the coverage statistics. + */ + return NULL; /* LCOV_EXCL_LINE */ + } for (; n > 0; --n, s++) { if (!poolAppendChar(pool, *s)) return NULL; @@ -6641,8 +6983,19 @@ poolGrow(STRING_POOL *pool) int blockSize = (int)((unsigned)(pool->end - pool->start)*2U); size_t bytesToAllocate; - if (blockSize < 0) - return XML_FALSE; + // NOTE: Needs to be calculated prior to calling `realloc` + // to avoid dangling pointers: + const ptrdiff_t offsetInsideBlock = pool->ptr - pool->start; + + if (blockSize < 0) { + /* This condition traps a situation where either more than + * INT_MAX/2 bytes have already been allocated. This isn't + * readily testable, since it is unlikely that an average + * machine will have that much memory, so we exclude it from the + * coverage statistics. + */ + return XML_FALSE; /* LCOV_EXCL_LINE */ + } bytesToAllocate = poolBytesToAllocateFor(blockSize); if (bytesToAllocate == 0) @@ -6654,7 +7007,7 @@ poolGrow(STRING_POOL *pool) return XML_FALSE; pool->blocks = temp; pool->blocks->size = blockSize; - pool->ptr = pool->blocks->s + (pool->ptr - pool->start); + pool->ptr = pool->blocks->s + offsetInsideBlock; pool->start = pool->blocks->s; pool->end = pool->start + blockSize; } @@ -6663,8 +7016,18 @@ poolGrow(STRING_POOL *pool) int blockSize = (int)(pool->end - pool->start); size_t bytesToAllocate; - if (blockSize < 0) - return XML_FALSE; + if (blockSize < 0) { + /* This condition traps a situation where either more than + * INT_MAX bytes have already been allocated (which is prevented + * by various pieces of program logic, not least this one, never + * mind the unlikelihood of actually having that much memory) or + * the pool control fields have been corrupted (which could + * conceivably happen in an extremely buggy user handler + * function). Either way it isn't readily testable, so we + * exclude it from the coverage statistics. + */ + return XML_FALSE; /* LCOV_EXCL_LINE */ + } if (blockSize < INIT_BLOCK_SIZE) blockSize = INIT_BLOCK_SIZE; @@ -6827,3 +7190,26 @@ getElementType(XML_Parser parser, } return ret; } + +static XML_Char * +copyString(const XML_Char *s, + const XML_Memory_Handling_Suite *memsuite) +{ + int charsRequired = 0; + XML_Char *result; + + /* First determine how long the string is */ + while (s[charsRequired] != 0) { + charsRequired++; + } + /* Include the terminator */ + charsRequired++; + + /* Now allocate space for the copy */ + result = memsuite->malloc_fcn(charsRequired * sizeof(XML_Char)); + if (result == NULL) + return NULL; + /* Copy the original into place */ + memcpy(result, s, charsRequired * sizeof(XML_Char)); + return result; +} diff --git a/Modules/expat/xmlrole.c b/Modules/expat/xmlrole.c index a7c56302796..c809ee51482 100644 --- a/Modules/expat/xmlrole.c +++ b/Modules/expat/xmlrole.c @@ -170,7 +170,14 @@ prolog1(PROLOG_STATE *state, case XML_TOK_COMMENT: return XML_ROLE_COMMENT; case XML_TOK_BOM: - return XML_ROLE_NONE; + /* This case can never arise. To reach this role function, the + * parse must have passed through prolog0 and therefore have had + * some form of input, even if only a space. At that point, a + * byte order mark is no longer a valid character (though + * technically it should be interpreted as a non-breaking space), + * so will be rejected by the tokenizing stages. + */ + return XML_ROLE_NONE; /* LCOV_EXCL_LINE */ case XML_TOK_DECL_OPEN: if (!XmlNameMatchesAscii(enc, ptr + 2 * MIN_BYTES_PER_CHAR(enc), @@ -1285,6 +1292,26 @@ declClose(PROLOG_STATE *state, return common(state, tok); } +/* This function will only be invoked if the internal logic of the + * parser has broken down. It is used in two cases: + * + * 1: When the XML prolog has been finished. At this point the + * processor (the parser level above these role handlers) should + * switch from prologProcessor to contentProcessor and reinitialise + * the handler function. + * + * 2: When an error has been detected (via common() below). At this + * point again the processor should be switched to errorProcessor, + * which will never call a handler. + * + * The result of this is that error() can only be called if the + * processor switch failed to happen, which is an internal error and + * therefore we shouldn't be able to provoke it simply by using the + * library. It is a necessary backstop, however, so we merely exclude + * it from the coverage statistics. + * + * LCOV_EXCL_START + */ static int PTRCALL error(PROLOG_STATE *UNUSED_P(state), int UNUSED_P(tok), @@ -1294,6 +1321,7 @@ error(PROLOG_STATE *UNUSED_P(state), { return XML_ROLE_NONE; } +/* LCOV_EXCL_STOP */ static int FASTCALL common(PROLOG_STATE *state, int tok) diff --git a/Modules/expat/xmltok.c b/Modules/expat/xmltok.c index cdf0720dd89..db4a5c8ca3e 100644 --- a/Modules/expat/xmltok.c +++ b/Modules/expat/xmltok.c @@ -1019,7 +1019,11 @@ streqci(const char *s1, const char *s2) if (ASCII_a <= c1 && c1 <= ASCII_z) c1 += ASCII_A - ASCII_a; if (ASCII_a <= c2 && c2 <= ASCII_z) - c2 += ASCII_A - ASCII_a; + /* The following line will never get executed. streqci() is + * only called from two places, both of which guarantee to put + * upper-case strings into s2. + */ + c2 += ASCII_A - ASCII_a; /* LCOV_EXCL_LINE */ if (c1 != c2) return 0; if (!c1) @@ -1291,7 +1295,7 @@ XmlUtf8Encode(int c, char *buf) }; if (c < 0) - return 0; + return 0; /* LCOV_EXCL_LINE: this case is always eliminated beforehand */ if (c < min2) { buf[0] = (char)(c | UTF8_cval1); return 1; @@ -1314,7 +1318,7 @@ XmlUtf8Encode(int c, char *buf) buf[3] = (char)((c & 0x3f) | 0x80); return 4; } - return 0; + return 0; /* LCOV_EXCL_LINE: this case too is eliminated before calling */ } int FASTCALL @@ -1465,6 +1469,9 @@ XmlInitUnknownEncoding(void *mem, else if (c < 0) { if (c < -4) return 0; + /* Multi-byte sequences need a converter function */ + if (!convert) + return 0; e->normal.type[i] = (unsigned char)(BT_LEAD2 - (c + 2)); e->utf8[i][0] = 0; e->utf16[i] = 0; diff --git a/Modules/expat/xmltok_impl.c b/Modules/expat/xmltok_impl.c index 5f779c0571b..4fa1ff679ce 100644 --- a/Modules/expat/xmltok_impl.c +++ b/Modules/expat/xmltok_impl.c @@ -1198,8 +1198,14 @@ PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr, const char *start; if (ptr >= end) return XML_TOK_NONE; - else if (! HAS_CHAR(enc, ptr, end)) - return XML_TOK_PARTIAL; + else if (! HAS_CHAR(enc, ptr, end)) { + /* This line cannot be executed. The incoming data has already + * been tokenized once, so incomplete characters like this have + * already been eliminated from the input. Retaining the paranoia + * check is still valuable, however. + */ + return XML_TOK_PARTIAL; /* LCOV_EXCL_LINE */ + } start = ptr; while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { @@ -1258,8 +1264,14 @@ PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr, const char *start; if (ptr >= end) return XML_TOK_NONE; - else if (! HAS_CHAR(enc, ptr, end)) - return XML_TOK_PARTIAL; + else if (! HAS_CHAR(enc, ptr, end)) { + /* This line cannot be executed. The incoming data has already + * been tokenized once, so incomplete characters like this have + * already been eliminated from the input. Retaining the paranoia + * check is still valuable, however. + */ + return XML_TOK_PARTIAL; /* LCOV_EXCL_LINE */ + } start = ptr; while (HAS_CHAR(enc, ptr, end)) { switch (BYTE_TYPE(enc, ptr)) { @@ -1614,6 +1626,14 @@ PREFIX(predefinedEntityName)(const ENCODING *UNUSED_P(enc), const char *ptr, return 0; } +/* This function does not appear to be called from anywhere within the + * library code. It is used via the macro XmlSameName(), which is + * defined but never used. Since it appears in the encoding function + * table, removing it is not a thing to be undertaken lightly. For + * the moment, we simply exclude it from coverage tests. + * + * LCOV_EXCL_START + */ static int PTRCALL PREFIX(sameName)(const ENCODING *enc, const char *ptr1, const char *ptr2) { @@ -1677,14 +1697,21 @@ PREFIX(sameName)(const ENCODING *enc, const char *ptr1, const char *ptr2) } /* not reached */ } +/* LCOV_EXCL_STOP */ static int PTRCALL PREFIX(nameMatchesAscii)(const ENCODING *UNUSED_P(enc), const char *ptr1, const char *end1, const char *ptr2) { for (; *ptr2; ptr1 += MINBPC(enc), ptr2++) { - if (end1 - ptr1 < MINBPC(enc)) - return 0; + if (end1 - ptr1 < MINBPC(enc)) { + /* This line cannot be executed. THe incoming data has already + * been tokenized once, so imcomplete characters like this have + * already been eliminated from the input. Retaining the + * paranoia check is still valuable, however. + */ + return 0; /* LCOV_EXCL_LINE */ + } if (!CHAR_MATCHES(enc, ptr1, *ptr2)) return 0; } diff --git a/PCbuild/_elementtree.vcxproj b/PCbuild/_elementtree.vcxproj index 639ba479db5..0db61f20818 100644 --- a/PCbuild/_elementtree.vcxproj +++ b/PCbuild/_elementtree.vcxproj @@ -87,6 +87,7 @@ </ItemGroup> <ItemGroup> <ClCompile Include="..\Modules\_elementtree.c" /> + <ClCompile Include="..\Modules\expat\loadlibrary.c" /> <ClCompile Include="..\Modules\expat\xmlparse.c" /> <ClCompile Include="..\Modules\expat\xmlrole.c" /> <ClCompile Include="..\Modules\expat\xmltok.c" /> diff --git a/PCbuild/_elementtree.vcxproj.filters b/PCbuild/_elementtree.vcxproj.filters index ee78295fa94..4597ee521b3 100644 --- a/PCbuild/_elementtree.vcxproj.filters +++ b/PCbuild/_elementtree.vcxproj.filters @@ -33,6 +33,9 @@ <ClInclude Include="..\Modules\expat\latin1tab.h"> <Filter>Header Files</Filter> </ClInclude> + <ClInclude Include="..\Modules\expat\loadlibrary.c"> + <Filter>Header Files</Filter> + </ClInclude> <ClInclude Include="..\Modules\expat\macconfig.h"> <Filter>Header Files</Filter> </ClInclude> @@ -69,4 +72,4 @@ <Filter>Source Files</Filter> </ClCompile> </ItemGroup> -</Project> \ No newline at end of file +</Project> diff --git a/PCbuild/pyexpat.vcxproj b/PCbuild/pyexpat.vcxproj index a2763f02063..51ca69e95d7 100644 --- a/PCbuild/pyexpat.vcxproj +++ b/PCbuild/pyexpat.vcxproj @@ -68,6 +68,7 @@ </ItemGroup> <ItemGroup> <ClCompile Include="..\Modules\pyexpat.c" /> + <ClCompile Include="..\Modules\expat\loadlibrary.c" /> <ClCompile Include="..\Modules\expat\xmlparse.c" /> <ClCompile Include="..\Modules\expat\xmlrole.c" /> <ClCompile Include="..\Modules\expat\xmltok.c" /> @@ -84,4 +85,4 @@ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <ImportGroup Label="ExtensionTargets"> </ImportGroup> -</Project> \ No newline at end of file +</Project> diff --git a/PCbuild/pyexpat.vcxproj.filters b/PCbuild/pyexpat.vcxproj.filters index abf362fb0bf..cb02847980c 100644 --- a/PCbuild/pyexpat.vcxproj.filters +++ b/PCbuild/pyexpat.vcxproj.filters @@ -20,6 +20,9 @@ <ClCompile Include="..\Modules\pyexpat.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\Modules\expat\loadlibrary.c"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="..\Modules\expat\xmlparse.c"> <Filter>Source Files</Filter> </ClCompile> @@ -30,4 +33,4 @@ <Filter>Source Files</Filter> </ClCompile> </ItemGroup> -</Project> \ No newline at end of file +</Project> diff --git a/setup.py b/setup.py index 41aad8210d8..f3c2538345e 100644 --- a/setup.py +++ b/setup.py @@ -1525,6 +1525,9 @@ class db_found(Exception): pass expat_inc = [os.path.join(os.getcwd(), srcdir, 'Modules', 'expat')] define_macros = [ ('HAVE_EXPAT_CONFIG_H', '1'), + # bpo-30947: Python uses best available entropy sources to + # call XML_SetHashSalt(), expat entropy sources are not needed + ('XML_POOR_ENTROPY', '1'), ] expat_lib = [] expat_sources = ['expat/xmlparse.c',

1 0

bpo-31234: fork_wait tests now join threads (#3139)
by Victor Stinner 19 Aug '17

19 Aug '17

https://github.com/python/cpython/commit/c99d41f9c0304fcf06550515c3db55f93a… commit: c99d41f9c0304fcf06550515c3db55f93a629e9e branch: master author: Victor Stinner <victor.stinner(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-08-18T23:12:26+02:00 summary: bpo-31234: fork_wait tests now join threads (#3139) fork_wait.py tests now joins threads, to not leak running threads in the background. files: M Lib/test/fork_wait.py diff --git a/Lib/test/fork_wait.py b/Lib/test/fork_wait.py index 713039dd826..6af79ad5d4d 100644 --- a/Lib/test/fork_wait.py +++ b/Lib/test/fork_wait.py @@ -11,7 +11,8 @@ import os, sys, time, unittest import test.support as support -_thread = support.import_module('_thread') + +threading = support.import_module('threading') LONGSLEEP = 2 SHORTSLEEP = 0.5 @@ -20,8 +21,19 @@ class ForkWait(unittest.TestCase): def setUp(self): + self._threading_key = support.threading_setup() self.alive = {} self.stop = 0 + self.threads = [] + + def tearDown(self): + # Stop threads + self.stop = 1 + for thread in self.threads: + thread.join() + thread = None + self.threads.clear() + support.threading_cleanup(*self._threading_key) def f(self, id): while not self.stop: @@ -43,10 +55,11 @@ def wait_impl(self, cpid): self.assertEqual(spid, cpid) self.assertEqual(status, 0, "cause = %d, exit = %d" % (status&0xff, status>>8)) - @support.reap_threads def test_wait(self): for i in range(NUM_THREADS): - _thread.start_new(self.f, (i,)) + thread = threading.Thread(target=self.f, args=(i,)) + thread.start() + self.threads.append(thread) # busy-loop to wait for threads deadline = time.monotonic() + 10.0 @@ -75,8 +88,4 @@ def test_wait(self): os._exit(n) else: # Parent - try: - self.wait_impl(cpid) - finally: - # Tell threads to die - self.stop = 1 + self.wait_impl(cpid)

1 0

bpo-30923: Disable warning that has been part of -Wextra since gcc-7.0. (#3142)
by Stefan Krah 19 Aug '17

19 Aug '17

https://github.com/python/cpython/commit/d73a960c575207539c3f9765cff26d4fff… commit: d73a960c575207539c3f9765cff26d4fff400b45 branch: master author: Stefan Krah <skrah(a)bytereef.org> committer: GitHub <noreply(a)github.com> date: 2017-08-18T21:39:32+02:00 summary: bpo-30923: Disable warning that has been part of -Wextra since gcc-7.0. (#3142) files: M Modules/_decimal/libmpdec/io.c diff --git a/Modules/_decimal/libmpdec/io.c b/Modules/_decimal/libmpdec/io.c index 3aadfb04379..f45e558f1a9 100644 --- a/Modules/_decimal/libmpdec/io.c +++ b/Modules/_decimal/libmpdec/io.c @@ -45,6 +45,12 @@ PEP-3101 formatting for numeric types. */ +/* Disable warning that is part of -Wextra since gcc 7.0. */ +#if defined(__GNUC__) && !defined(__INTEL_COMPILER) && __GNUC__ >= 7 + #pragma GCC diagnostic ignored "-Wimplicit-fallthrough" +#endif + + /* * Work around the behavior of tolower() and strcasecmp() in certain * locales. For example, in tr_TR.utf8:

1 0

bpo-31109: Convert zipimport to use Argument Clinic (GH-2990)
by Brett Cannon 19 Aug '17

19 Aug '17

https://github.com/python/cpython/commit/02f3b7d5ab2206d256879e5a8a34f56021… commit: 02f3b7d5ab2206d256879e5a8a34f560218ed397 branch: master author: Yaron de Leeuw <me(a)jarondl.net> committer: Brett Cannon <brettcannon(a)users.noreply.github.com> date: 2017-08-18T11:41:13-07:00 summary: bpo-31109: Convert zipimport to use Argument Clinic (GH-2990) files: A Misc/NEWS.d/next/Library/2017-08-17-20-29-45.bpo-31109.7qtC64.rst A Modules/clinic/zipimport.c.h M Misc/ACKS M Modules/zipimport.c diff --git a/Misc/ACKS b/Misc/ACKS index 6fc41b36b59..21fc6bbbd3a 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -889,6 +889,7 @@ James Lee John J. Lee Thomas Lee Cooper Ry Lees +Yaron de Leeuw Tennessee Leeuwenburg Luc Lefebvre Pierre Paul Lefebvre diff --git a/Misc/NEWS.d/next/Library/2017-08-17-20-29-45.bpo-31109.7qtC64.rst b/Misc/NEWS.d/next/Library/2017-08-17-20-29-45.bpo-31109.7qtC64.rst new file mode 100644 index 00000000000..f023cbc80eb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-08-17-20-29-45.bpo-31109.7qtC64.rst @@ -0,0 +1 @@ +Convert zipimport to use Argument Clinic. diff --git a/Modules/clinic/zipimport.c.h b/Modules/clinic/zipimport.c.h new file mode 100644 index 00000000000..b24838d73b7 --- /dev/null +++ b/Modules/clinic/zipimport.c.h @@ -0,0 +1,294 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +PyDoc_STRVAR(zipimport_zipimporter___init____doc__, +"zipimporter(archivepath, /)\n" +"--\n" +"\n" +"Create a new zipimporter instance.\n" +"\n" +" archivepath\n" +" A path-like object to a zipfile, or to a specific path inside\n" +" a zipfile.\n" +"\n" +"\'archivepath\' must be a path-like object to a zipfile, or to a specific path\n" +"inside a zipfile. For example, it can be \'/tmp/myimport.zip\', or\n" +"\'/tmp/myimport.zip/mydirectory\', if mydirectory is a valid directory inside\n" +"the archive.\n" +"\n" +"\'ZipImportError\' is raised if \'archivepath\' doesn\'t point to a valid Zip\n" +"archive.\n" +"\n" +"The \'archive\' attribute of the zipimporter object contains the name of the\n" +"zipfile targeted."); + +static int +zipimport_zipimporter___init___impl(ZipImporter *self, PyObject *path); + +static int +zipimport_zipimporter___init__(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + PyObject *path; + + if ((Py_TYPE(self) == &ZipImporter_Type) && + !_PyArg_NoKeywords("zipimporter", kwargs)) { + goto exit; + } + if (!PyArg_ParseTuple(args, "O&:zipimporter", + PyUnicode_FSDecoder, &path)) { + goto exit; + } + return_value = zipimport_zipimporter___init___impl((ZipImporter *)self, path); + +exit: + return return_value; +} + +PyDoc_STRVAR(zipimport_zipimporter_find_module__doc__, +"find_module($self, fullname, path=None, /)\n" +"--\n" +"\n" +"Search for a module specified by \'fullname\'.\n" +"\n" +"\'fullname\' must be the fully qualified (dotted) module name. It returns the\n" +"zipimporter instance itself if the module was found, or None if it wasn\'t.\n" +"The optional \'path\' argument is ignored -- it\'s there for compatibility\n" +"with the importer protocol."); + +#define ZIPIMPORT_ZIPIMPORTER_FIND_MODULE_METHODDEF \ + {"find_module", (PyCFunction)zipimport_zipimporter_find_module, METH_FASTCALL, zipimport_zipimporter_find_module__doc__}, + +static PyObject * +zipimport_zipimporter_find_module_impl(ZipImporter *self, PyObject *fullname, + PyObject *path); + +static PyObject * +zipimport_zipimporter_find_module(ZipImporter *self, PyObject **args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *fullname; + PyObject *path = Py_None; + + if (!_PyArg_ParseStack(args, nargs, "U|O:find_module", + &fullname, &path)) { + goto exit; + } + return_value = zipimport_zipimporter_find_module_impl(self, fullname, path); + +exit: + return return_value; +} + +PyDoc_STRVAR(zipimport_zipimporter_find_loader__doc__, +"find_loader($self, fullname, path=None, /)\n" +"--\n" +"\n" +"Search for a module specified by \'fullname\'.\n" +"\n" +"\'fullname\' must be the fully qualified (dotted) module name. It returns the\n" +"zipimporter instance itself if the module was found, a string containing the\n" +"full path name if it\'s possibly a portion of a namespace package,\n" +"or None otherwise. The optional \'path\' argument is ignored -- it\'s\n" +"there for compatibility with the importer protocol."); + +#define ZIPIMPORT_ZIPIMPORTER_FIND_LOADER_METHODDEF \ + {"find_loader", (PyCFunction)zipimport_zipimporter_find_loader, METH_FASTCALL, zipimport_zipimporter_find_loader__doc__}, + +static PyObject * +zipimport_zipimporter_find_loader_impl(ZipImporter *self, PyObject *fullname, + PyObject *path); + +static PyObject * +zipimport_zipimporter_find_loader(ZipImporter *self, PyObject **args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *fullname; + PyObject *path = Py_None; + + if (!_PyArg_ParseStack(args, nargs, "U|O:find_loader", + &fullname, &path)) { + goto exit; + } + return_value = zipimport_zipimporter_find_loader_impl(self, fullname, path); + +exit: + return return_value; +} + +PyDoc_STRVAR(zipimport_zipimporter_load_module__doc__, +"load_module($self, fullname, /)\n" +"--\n" +"\n" +"Load the module specified by \'fullname\'.\n" +"\n" +"\'fullname\' must be the fully qualified (dotted) module name. It returns the\n" +"imported module, or raises ZipImportError if it wasn\'t found."); + +#define ZIPIMPORT_ZIPIMPORTER_LOAD_MODULE_METHODDEF \ + {"load_module", (PyCFunction)zipimport_zipimporter_load_module, METH_O, zipimport_zipimporter_load_module__doc__}, + +static PyObject * +zipimport_zipimporter_load_module_impl(ZipImporter *self, PyObject *fullname); + +static PyObject * +zipimport_zipimporter_load_module(ZipImporter *self, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *fullname; + + if (!PyArg_Parse(arg, "U:load_module", &fullname)) { + goto exit; + } + return_value = zipimport_zipimporter_load_module_impl(self, fullname); + +exit: + return return_value; +} + +PyDoc_STRVAR(zipimport_zipimporter_get_filename__doc__, +"get_filename($self, fullname, /)\n" +"--\n" +"\n" +"Return the filename for the specified module."); + +#define ZIPIMPORT_ZIPIMPORTER_GET_FILENAME_METHODDEF \ + {"get_filename", (PyCFunction)zipimport_zipimporter_get_filename, METH_O, zipimport_zipimporter_get_filename__doc__}, + +static PyObject * +zipimport_zipimporter_get_filename_impl(ZipImporter *self, + PyObject *fullname); + +static PyObject * +zipimport_zipimporter_get_filename(ZipImporter *self, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *fullname; + + if (!PyArg_Parse(arg, "U:get_filename", &fullname)) { + goto exit; + } + return_value = zipimport_zipimporter_get_filename_impl(self, fullname); + +exit: + return return_value; +} + +PyDoc_STRVAR(zipimport_zipimporter_is_package__doc__, +"is_package($self, fullname, /)\n" +"--\n" +"\n" +"Return True if the module specified by fullname is a package.\n" +"\n" +"Raise ZipImportError if the module couldn\'t be found."); + +#define ZIPIMPORT_ZIPIMPORTER_IS_PACKAGE_METHODDEF \ + {"is_package", (PyCFunction)zipimport_zipimporter_is_package, METH_O, zipimport_zipimporter_is_package__doc__}, + +static PyObject * +zipimport_zipimporter_is_package_impl(ZipImporter *self, PyObject *fullname); + +static PyObject * +zipimport_zipimporter_is_package(ZipImporter *self, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *fullname; + + if (!PyArg_Parse(arg, "U:is_package", &fullname)) { + goto exit; + } + return_value = zipimport_zipimporter_is_package_impl(self, fullname); + +exit: + return return_value; +} + +PyDoc_STRVAR(zipimport_zipimporter_get_data__doc__, +"get_data($self, pathname, /)\n" +"--\n" +"\n" +"Return the data associated with \'pathname\'.\n" +"\n" +"Raise OSError if the file was not found."); + +#define ZIPIMPORT_ZIPIMPORTER_GET_DATA_METHODDEF \ + {"get_data", (PyCFunction)zipimport_zipimporter_get_data, METH_O, zipimport_zipimporter_get_data__doc__}, + +static PyObject * +zipimport_zipimporter_get_data_impl(ZipImporter *self, PyObject *path); + +static PyObject * +zipimport_zipimporter_get_data(ZipImporter *self, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *path; + + if (!PyArg_Parse(arg, "U:get_data", &path)) { + goto exit; + } + return_value = zipimport_zipimporter_get_data_impl(self, path); + +exit: + return return_value; +} + +PyDoc_STRVAR(zipimport_zipimporter_get_code__doc__, +"get_code($self, fullname, /)\n" +"--\n" +"\n" +"Return the code object for the specified module.\n" +"\n" +"Raise ZipImportError if the module couldn\'t be found."); + +#define ZIPIMPORT_ZIPIMPORTER_GET_CODE_METHODDEF \ + {"get_code", (PyCFunction)zipimport_zipimporter_get_code, METH_O, zipimport_zipimporter_get_code__doc__}, + +static PyObject * +zipimport_zipimporter_get_code_impl(ZipImporter *self, PyObject *fullname); + +static PyObject * +zipimport_zipimporter_get_code(ZipImporter *self, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *fullname; + + if (!PyArg_Parse(arg, "U:get_code", &fullname)) { + goto exit; + } + return_value = zipimport_zipimporter_get_code_impl(self, fullname); + +exit: + return return_value; +} + +PyDoc_STRVAR(zipimport_zipimporter_get_source__doc__, +"get_source($self, fullname, /)\n" +"--\n" +"\n" +"Return the source code for the specified module.\n" +"\n" +"Raise ZipImportError if the module couldn\'t be found, return None if the\n" +"archive does contain the module, but has no source for it."); + +#define ZIPIMPORT_ZIPIMPORTER_GET_SOURCE_METHODDEF \ + {"get_source", (PyCFunction)zipimport_zipimporter_get_source, METH_O, zipimport_zipimporter_get_source__doc__}, + +static PyObject * +zipimport_zipimporter_get_source_impl(ZipImporter *self, PyObject *fullname); + +static PyObject * +zipimport_zipimporter_get_source(ZipImporter *self, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *fullname; + + if (!PyArg_Parse(arg, "U:get_source", &fullname)) { + goto exit; + } + return_value = zipimport_zipimporter_get_source_impl(self, fullname); + +exit: + return return_value; +} +/*[clinic end generated code: output=bac6c9144950eaec input=a9049054013a1b77]*/ diff --git a/Modules/zipimport.c b/Modules/zipimport.c index fad1b1f5abb..c01ee55840b 100644 --- a/Modules/zipimport.c +++ b/Modules/zipimport.c @@ -54,28 +54,54 @@ static PyObject *get_data(PyObject *archive, PyObject *toc_entry); static PyObject *get_module_code(ZipImporter *self, PyObject *fullname, int *p_ispackage, PyObject **p_modpath); +static PyTypeObject ZipImporter_Type; #define ZipImporter_Check(op) PyObject_TypeCheck(op, &ZipImporter_Type) +/*[clinic input] +module zipimport +class zipimport.zipimporter "ZipImporter *" "&ZipImporter_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=9db8b61557d911e7]*/ +#include "clinic/zipimport.c.h" + /* zipimporter.__init__ Split the "subdirectory" from the Zip archive path, lookup a matching entry in sys.path_importer_cache, fetch the file directory from there if found, or else read it from the archive. */ + +/*[clinic input] +zipimport.zipimporter.__init__ + + archivepath as path: object(converter="PyUnicode_FSDecoder") + A path-like object to a zipfile, or to a specific path inside + a zipfile. + / + +Create a new zipimporter instance. + +'archivepath' must be a path-like object to a zipfile, or to a specific path +inside a zipfile. For example, it can be '/tmp/myimport.zip', or +'/tmp/myimport.zip/mydirectory', if mydirectory is a valid directory inside +the archive. + +'ZipImportError' is raised if 'archivepath' doesn't point to a valid Zip +archive. + +The 'archive' attribute of the zipimporter object contains the name of the +zipfile targeted. + +[clinic start generated code]*/ + static int -zipimporter_init(ZipImporter *self, PyObject *args, PyObject *kwds) +zipimport_zipimporter___init___impl(ZipImporter *self, PyObject *path) +/*[clinic end generated code: output=141558fefdb46dc8 input=92b9ebeed1f6a704]*/ { - PyObject *path, *files, *tmp; + PyObject *files, *tmp; PyObject *filename = NULL; Py_ssize_t len, flen; - if (!_PyArg_NoKeywords("zipimporter", kwds)) - return -1; - - if (!PyArg_ParseTuple(args, "O&:zipimporter", - PyUnicode_FSDecoder, &path)) - return -1; - if (PyUnicode_READY(path) == -1) return -1; @@ -379,21 +405,30 @@ find_loader(ZipImporter *self, PyObject *fullname, PyObject **namespace_portion) return FL_MODULE_FOUND; } +/*[clinic input] +zipimport.zipimporter.find_module + + fullname: unicode + path: object = None + / + +Search for a module specified by 'fullname'. + +'fullname' must be the fully qualified (dotted) module name. It returns the +zipimporter instance itself if the module was found, or None if it wasn't. +The optional 'path' argument is ignored -- it's there for compatibility +with the importer protocol. + +[clinic start generated code]*/ -/* Check whether we can satisfy the import of the module named by - 'fullname'. Return self if we can, None if we can't. */ static PyObject * -zipimporter_find_module(PyObject *obj, PyObject *args) +zipimport_zipimporter_find_module_impl(ZipImporter *self, PyObject *fullname, + PyObject *path) +/*[clinic end generated code: output=506087f609466dc7 input=e3528520e075063f]*/ { - ZipImporter *self = (ZipImporter *)obj; - PyObject *path = NULL; - PyObject *fullname; PyObject *namespace_portion = NULL; PyObject *result = NULL; - if (!PyArg_ParseTuple(args, "U|O:zipimporter.find_module", &fullname, &path)) - return NULL; - switch (find_loader(self, fullname, &namespace_portion)) { case FL_ERROR: return NULL; @@ -416,23 +451,31 @@ zipimporter_find_module(PyObject *obj, PyObject *args) } -/* Check whether we can satisfy the import of the module named by - 'fullname', or whether it could be a portion of a namespace - package. Return self if we can load it, a string containing the - full path if it's a possible namespace portion, None if we - can't load it. */ +/*[clinic input] +zipimport.zipimporter.find_loader + + fullname: unicode + path: object = None + / + +Search for a module specified by 'fullname'. + +'fullname' must be the fully qualified (dotted) module name. It returns the +zipimporter instance itself if the module was found, a string containing the +full path name if it's possibly a portion of a namespace package, +or None otherwise. The optional 'path' argument is ignored -- it's +there for compatibility with the importer protocol. + +[clinic start generated code]*/ + static PyObject * -zipimporter_find_loader(PyObject *obj, PyObject *args) +zipimport_zipimporter_find_loader_impl(ZipImporter *self, PyObject *fullname, + PyObject *path) +/*[clinic end generated code: output=601599a43bc0f49a input=dc73f275b0d5be23]*/ { - ZipImporter *self = (ZipImporter *)obj; - PyObject *path = NULL; - PyObject *fullname; PyObject *result = NULL; PyObject *namespace_portion = NULL; - if (!PyArg_ParseTuple(args, "U|O:zipimporter.find_module", &fullname, &path)) - return NULL; - switch (find_loader(self, fullname, &namespace_portion)) { case FL_ERROR: return NULL; @@ -453,19 +496,27 @@ zipimporter_find_loader(PyObject *obj, PyObject *args) return result; } -/* Load and return the module named by 'fullname'. */ +/*[clinic input] +zipimport.zipimporter.load_module + + fullname: unicode + / + +Load the module specified by 'fullname'. + +'fullname' must be the fully qualified (dotted) module name. It returns the +imported module, or raises ZipImportError if it wasn't found. + +[clinic start generated code]*/ + static PyObject * -zipimporter_load_module(PyObject *obj, PyObject *args) +zipimport_zipimporter_load_module_impl(ZipImporter *self, PyObject *fullname) +/*[clinic end generated code: output=7303cebf88d47953 input=c236e2e8621f04ef]*/ { - ZipImporter *self = (ZipImporter *)obj; PyObject *code = NULL, *mod, *dict; - PyObject *fullname; PyObject *modpath = NULL; int ispackage; - if (!PyArg_ParseTuple(args, "U:zipimporter.load_module", - &fullname)) - return NULL; if (PyUnicode_READY(fullname) == -1) return NULL; @@ -523,18 +574,23 @@ zipimporter_load_module(PyObject *obj, PyObject *args) return NULL; } -/* Return a string matching __file__ for the named module */ +/*[clinic input] +zipimport.zipimporter.get_filename + + fullname: unicode + / + +Return the filename for the specified module. +[clinic start generated code]*/ + static PyObject * -zipimporter_get_filename(PyObject *obj, PyObject *args) +zipimport_zipimporter_get_filename_impl(ZipImporter *self, + PyObject *fullname) +/*[clinic end generated code: output=c5b92b58bea86506 input=28d2eb57e4f25c8a]*/ { - ZipImporter *self = (ZipImporter *)obj; - PyObject *fullname, *code, *modpath; + PyObject *code, *modpath; int ispackage; - if (!PyArg_ParseTuple(args, "U:zipimporter.get_filename", - &fullname)) - return NULL; - /* Deciding the filename requires working out where the code would come from if the module was actually loaded */ code = get_module_code(self, fullname, &ispackage, &modpath); @@ -545,18 +601,24 @@ zipimporter_get_filename(PyObject *obj, PyObject *args) return modpath; } -/* Return a bool signifying whether the module is a package or not. */ +/*[clinic input] +zipimport.zipimporter.is_package + + fullname: unicode + / + +Return True if the module specified by fullname is a package. + +Raise ZipImportError if the module couldn't be found. + +[clinic start generated code]*/ + static PyObject * -zipimporter_is_package(PyObject *obj, PyObject *args) +zipimport_zipimporter_is_package_impl(ZipImporter *self, PyObject *fullname) +/*[clinic end generated code: output=c32958c2a5216ae6 input=a7ba752f64345062]*/ { - ZipImporter *self = (ZipImporter *)obj; - PyObject *fullname; enum zi_module_info mi; - if (!PyArg_ParseTuple(args, "U:zipimporter.is_package", - &fullname)) - return NULL; - mi = get_module_info(self, fullname); if (mi == MI_ERROR) return NULL; @@ -568,17 +630,26 @@ zipimporter_is_package(PyObject *obj, PyObject *args) } +/*[clinic input] +zipimport.zipimporter.get_data + + pathname as path: unicode + / + +Return the data associated with 'pathname'. + +Raise OSError if the file was not found. + +[clinic start generated code]*/ + static PyObject * -zipimporter_get_data(PyObject *obj, PyObject *args) +zipimport_zipimporter_get_data_impl(ZipImporter *self, PyObject *path) +/*[clinic end generated code: output=65dc506aaa268436 input=fa6428b74843c4ae]*/ { - ZipImporter *self = (ZipImporter *)obj; - PyObject *path, *key; + PyObject *key; PyObject *toc_entry; Py_ssize_t path_start, path_len, len; - if (!PyArg_ParseTuple(args, "U:zipimporter.get_data", &path)) - return NULL; - #ifdef ALTSEP path = _PyObject_CallMethodId(path, &PyId_replace, "CC", ALTSEP, SEP); if (!path) @@ -615,29 +686,46 @@ zipimporter_get_data(PyObject *obj, PyObject *args) return NULL; } -static PyObject * -zipimporter_get_code(PyObject *obj, PyObject *args) -{ - ZipImporter *self = (ZipImporter *)obj; - PyObject *fullname; +/*[clinic input] +zipimport.zipimporter.get_code - if (!PyArg_ParseTuple(args, "U:zipimporter.get_code", &fullname)) - return NULL; + fullname: unicode + / + +Return the code object for the specified module. + +Raise ZipImportError if the module couldn't be found. + +[clinic start generated code]*/ +static PyObject * +zipimport_zipimporter_get_code_impl(ZipImporter *self, PyObject *fullname) +/*[clinic end generated code: output=b923c37fa99cbac4 input=2761412bc37f3549]*/ +{ return get_module_code(self, fullname, NULL, NULL); } +/*[clinic input] +zipimport.zipimporter.get_source + + fullname: unicode + / + +Return the source code for the specified module. + +Raise ZipImportError if the module couldn't be found, return None if the +archive does contain the module, but has no source for it. + +[clinic start generated code]*/ + static PyObject * -zipimporter_get_source(PyObject *obj, PyObject *args) +zipimport_zipimporter_get_source_impl(ZipImporter *self, PyObject *fullname) +/*[clinic end generated code: output=bc059301b0c33729 input=4e4b186f2e690716]*/ { - ZipImporter *self = (ZipImporter *)obj; PyObject *toc_entry; - PyObject *fullname, *subname, *path, *fullpath; + PyObject *subname, *path, *fullpath; enum zi_module_info mi; - if (!PyArg_ParseTuple(args, "U:zipimporter.get_source", &fullname)) - return NULL; - mi = get_module_info(self, fullname); if (mi == MI_ERROR) return NULL; @@ -680,80 +768,16 @@ zipimporter_get_source(PyObject *obj, PyObject *args) Py_RETURN_NONE; } -PyDoc_STRVAR(doc_find_module, -"find_module(fullname, path=None) -> self or None.\n\ -\n\ -Search for a module specified by 'fullname'. 'fullname' must be the\n\ -fully qualified (dotted) module name. It returns the zipimporter\n\ -instance itself if the module was found, or None if it wasn't.\n\ -The optional 'path' argument is ignored -- it's there for compatibility\n\ -with the importer protocol."); - -PyDoc_STRVAR(doc_find_loader, -"find_loader(fullname, path=None) -> self, str or None.\n\ -\n\ -Search for a module specified by 'fullname'. 'fullname' must be the\n\ -fully qualified (dotted) module name. It returns the zipimporter\n\ -instance itself if the module was found, a string containing the\n\ -full path name if it's possibly a portion of a namespace package,\n\ -or None otherwise. The optional 'path' argument is ignored -- it's\n\ - there for compatibility with the importer protocol."); - -PyDoc_STRVAR(doc_load_module, -"load_module(fullname) -> module.\n\ -\n\ -Load the module specified by 'fullname'. 'fullname' must be the\n\ -fully qualified (dotted) module name. It returns the imported\n\ -module, or raises ZipImportError if it wasn't found."); - -PyDoc_STRVAR(doc_get_data, -"get_data(pathname) -> string with file data.\n\ -\n\ -Return the data associated with 'pathname'. Raise OSError if\n\ -the file wasn't found."); - -PyDoc_STRVAR(doc_is_package, -"is_package(fullname) -> bool.\n\ -\n\ -Return True if the module specified by fullname is a package.\n\ -Raise ZipImportError if the module couldn't be found."); - -PyDoc_STRVAR(doc_get_code, -"get_code(fullname) -> code object.\n\ -\n\ -Return the code object for the specified module. Raise ZipImportError\n\ -if the module couldn't be found."); - -PyDoc_STRVAR(doc_get_source, -"get_source(fullname) -> source string.\n\ -\n\ -Return the source code for the specified module. Raise ZipImportError\n\ -if the module couldn't be found, return None if the archive does\n\ -contain the module, but has no source for it."); - - -PyDoc_STRVAR(doc_get_filename, -"get_filename(fullname) -> filename string.\n\ -\n\ -Return the filename for the specified module."); static PyMethodDef zipimporter_methods[] = { - {"find_module", zipimporter_find_module, METH_VARARGS, - doc_find_module}, - {"find_loader", zipimporter_find_loader, METH_VARARGS, - doc_find_loader}, - {"load_module", zipimporter_load_module, METH_VARARGS, - doc_load_module}, - {"get_data", zipimporter_get_data, METH_VARARGS, - doc_get_data}, - {"get_code", zipimporter_get_code, METH_VARARGS, - doc_get_code}, - {"get_source", zipimporter_get_source, METH_VARARGS, - doc_get_source}, - {"get_filename", zipimporter_get_filename, METH_VARARGS, - doc_get_filename}, - {"is_package", zipimporter_is_package, METH_VARARGS, - doc_is_package}, + ZIPIMPORT_ZIPIMPORTER_FIND_MODULE_METHODDEF + ZIPIMPORT_ZIPIMPORTER_FIND_LOADER_METHODDEF + ZIPIMPORT_ZIPIMPORTER_LOAD_MODULE_METHODDEF + ZIPIMPORT_ZIPIMPORTER_GET_FILENAME_METHODDEF + ZIPIMPORT_ZIPIMPORTER_IS_PACKAGE_METHODDEF + ZIPIMPORT_ZIPIMPORTER_GET_DATA_METHODDEF + ZIPIMPORT_ZIPIMPORTER_GET_CODE_METHODDEF + ZIPIMPORT_ZIPIMPORTER_GET_SOURCE_METHODDEF {NULL, NULL} /* sentinel */ }; @@ -764,20 +788,6 @@ static PyMemberDef zipimporter_members[] = { {NULL} }; -PyDoc_STRVAR(zipimporter_doc, -"zipimporter(archivepath) -> zipimporter object\n\ -\n\ -Create a new zipimporter instance. 'archivepath' must be a path to\n\ -a zipfile, or to a specific path inside a zipfile. For example, it can be\n\ -'/tmp/myimport.zip', or '/tmp/myimport.zip/mydirectory', if mydirectory is a\n\ -valid directory inside the archive.\n\ -\n\ -'ZipImportError is raised if 'archivepath' doesn't point to a valid Zip\n\ -archive.\n\ -\n\ -The 'archive' attribute of zipimporter objects contains the name of the\n\ -zipfile targeted."); - #define DEFERRED_ADDRESS(ADDR) 0 static PyTypeObject ZipImporter_Type = { @@ -802,7 +812,7 @@ static PyTypeObject ZipImporter_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - zipimporter_doc, /* tp_doc */ + zipimport_zipimporter___init____doc__, /* tp_doc */ zipimporter_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ @@ -817,7 +827,7 @@ static PyTypeObject ZipImporter_Type = { 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ - (initproc)zipimporter_init, /* tp_init */ + (initproc)zipimport_zipimporter___init__, /* tp_init */ PyType_GenericAlloc, /* tp_alloc */ PyType_GenericNew, /* tp_new */ PyObject_GC_Del, /* tp_free */

1 0

[65 flat] Results for Python (master branch) 2017-08-18
by lp_benchmark_robot＠intel.com 18 Aug '17

18 Aug '17

Results for project python/master, build date: 2017-08-18 03:04:03-07:00. - commit: a3a01a2 - previous commit: ee84a60 - revision date: 2017-08-18 11:31:52+02:00 - environment: Broadwell-EP - cpu: Intel(R) Xeon(R) CPU E5-2699 v4 @ 2.20GHz 2x22 cores, stepping 1, LLC 55 MB - mem: 128 GB - os: Ubuntu 16.04.2 LTS - kernel: 4.4.0-62-generic x86_64 GNU/Linux Baseline results were generated using release v3.6.0, with hash 5c4568a from 2016-12-22 23:38:47+00:00. +-----+------------------------+--------+------------+------------+------------+ | | |relative|change since|change since|current rev | | | benchmark|std_dev*| last run | baseline |run with PGO| +-----+------------------------+--------+------------+------------+------------+ | :-| | 2to3| 1.207% | -0.059% | +4.061% | +8.219% | +-----+------------------------+--------+------------+------------+------------+ | :-| | call_method| 2.232% | +0.629% | +20.835% | +10.529% | +-----+------------------------+--------+------------+------------+------------+ | :-| | call_method_slots| 2.391% | -0.224% | +23.649% | +7.481% | +-----+------------------------+--------+------------+------------+------------+ | :-| | call_method_unknown| 0.498% | +0.138% | +21.079% | +7.175% | +-----+------------------------+--------+------------+------------+------------+ | :-| | call_simple| 2.646% | -0.390% | +1.928% | +15.335% | +-----+------------------------+--------+------------+------------+------------+ | :-| | chameleon| 1.323% | +0.155% | +11.686% | +9.373% | +-----+------------------------+--------+------------+------------+------------+ | :-| | chaos| 1.547% | -0.483% | +5.768% | +11.791% | +-----+------------------------+--------+------------+------------+------------+ | :-| | crypto_pyaes| 0.530% | -0.028% | +4.486% | +4.822% | +-----+------------------------+--------+------------+------------+------------+ | :-| | deltablue| 4.103% | -0.101% | +5.922% | +19.353% | +-----+------------------------+--------+------------+------------+------------+ | :-| | django_template| 4.419% | +0.252% | +8.795% | +15.835% | +-----+------------------------+--------+------------+------------+------------+ | :-| | dulwich_log| 1.465% | +0.071% | +4.776% | +6.114% | +-----+------------------------+--------+------------+------------+------------+ | :-| | fannkuch| 0.496% | -0.050% | +4.442% | +4.362% | +-----+------------------------+--------+------------+------------+------------+ | :-| | float| 0.584% | -0.038% | +3.150% | +6.909% | +-----+------------------------+--------+------------+------------+------------+ | :-| | genshi_text| 1.627% | +0.214% | +8.890% | +10.063% | +-----+------------------------+--------+------------+------------+------------+ | :-| | genshi_xml| 2.289% | -0.523% | +7.519% | +8.801% | +-----+------------------------+--------+------------+------------+------------+ | :-| | go| 0.784% | +0.201% | +6.342% | +10.954% | +-----+------------------------+--------+------------+------------+------------+ | :-| | hexiom| 1.303% | -0.377% | +8.802% | +11.731% | +-----+------------------------+--------+------------+------------+------------+ | :-| | html5lib| 3.417% | +0.328% | +8.211% | +10.496% | +-----+------------------------+--------+------------+------------+------------+ | :-| | json_dumps| 2.645% | -0.118% | +4.154% | +9.960% | +-----+------------------------+--------+------------+------------+------------+ | :-| | json_loads| 1.331% | +0.312% | +3.136% | +10.087% | +-----+------------------------+--------+------------+------------+------------+ | :-| | logging_format| 1.853% | -0.969% | +8.063% | +11.832% | +-----+------------------------+--------+------------+------------+------------+ | :-| | logging_silent| 3.155% | -0.046% | +46.778% | +11.239% | +-----+------------------------+--------+------------+------------+------------+ | :-| | logging_simple| 1.965% | -0.716% | +9.038% | +12.497% | +-----+------------------------+--------+------------+------------+------------+ | :-| | mako| 0.595% | -0.101% | +18.768% | +10.676% | +-----+------------------------+--------+------------+------------+------------+ | :-| | mdp| 6.250% | -1.086% | +5.154% | +14.305% | +-----+------------------------+--------+------------+------------+------------+ | :-| | meteor_contest| 2.278% | +0.182% | +3.694% | +4.724% | +-----+------------------------+--------+------------+------------+------------+ | :-| | nbody| 0.396% | +0.109% | -1.994% | +2.189% | +-----+------------------------+--------+------------+------------+------------+ | :-| | nqueens| 0.964% | -0.074% | +2.713% | +6.658% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pathlib| 1.812% | -0.096% | +4.895% | +8.628% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pickle| 1.042% | +0.878% | +1.331% | +22.468% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pickle_dict| 0.462% | +0.024% | +2.564% | +15.808% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pickle_list| 0.829% | -0.015% | +5.600% | +16.694% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pickle_pure_python| 3.597% | +2.002% | +12.192% | +9.234% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pidigits| 0.066% | +0.011% | +0.322% | +9.999% | +-----+------------------------+--------+------------+------------+------------+ | :-| | python_startup| 0.137% | -0.091% | +8.937% | +4.755% | +-----+------------------------+--------+------------+------------+------------+ | :-| | python_startup_no_site| 0.095% | +0.036% | +1.068% | +4.358% | +-----+------------------------+--------+------------+------------+------------+ | :-| | raytrace| 1.194% | -0.371% | +8.992% | +13.462% | +-----+------------------------+--------+------------+------------+------------+ | :-| | regex_compile| 4.699% | +0.833% | -11.589% | +13.776% | +-----+------------------------+--------+------------+------------+------------+ | :-| | regex_dna| 0.632% | +0.052% | +1.542% | +9.376% | +-----+------------------------+--------+------------+------------+------------+ | :-| | regex_effbot| 2.006% | -0.106% | +0.426% | +1.203% | +-----+------------------------+--------+------------+------------+------------+ | :-| | regex_v8| 0.788% | -0.170% | +11.621% | +3.634% | +-----+------------------------+--------+------------+------------+------------+ | :-| | richards| 1.293% | +0.240% | +8.025% | +14.607% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_fft| 1.060% | -0.628% | -0.027% | +2.182% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_lu| 2.018% | +0.502% | +26.893% | +8.027% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_monte_carlo| 1.609% | -0.396% | +2.552% | +8.189% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_sor| 0.946% | -0.507% | +12.687% | +10.436% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_sparse_mat_mult| 1.391% | +0.820% | +0.470% | -3.308% | +-----+------------------------+--------+------------+------------+------------+ | :-| | spectral_norm| 0.343% | +0.097% | +5.747% | +1.955% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sqlalchemy_declarative| 1.861% | +1.090% | +6.520% | +7.745% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sqlalchemy_imperative| 2.641% | +0.986% | +6.390% | +5.319% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sqlite_synth| 4.373% | -2.146% | +2.257% | +9.919% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sympy_expand| 2.083% | +0.520% | +12.784% | +8.094% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sympy_integrate| 1.690% | -0.025% | +10.286% | +6.802% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sympy_str| 4.325% | -0.581% | +11.624% | +9.946% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sympy_sum| 4.404% | +0.790% | +14.083% | +9.931% | +-----+------------------------+--------+------------+------------+------------+ | :-| | telco| 3.036% | +0.392% | +23.838% | +6.918% | +-----+------------------------+--------+------------+------------+------------+ | :-| | tornado_http| 1.215% | +0.091% | +6.144% | +6.371% | +-----+------------------------+--------+------------+------------+------------+ | :-| | unpack_sequence| 1.365% | +0.335% | +0.719% | +2.764% | +-----+------------------------+--------+------------+------------+------------+ | :-| | unpickle|10.062% | -1.493% | +6.045% | +22.676% | +-----+------------------------+--------+------------+------------+------------+ | :-| | unpickle_list| 1.056% | -0.162% | -2.939% | +18.891% | +-----+------------------------+--------+------------+------------+------------+ | :-| | unpickle_pure_python| 1.191% | +0.351% | +7.324% | +5.696% | +-----+------------------------+--------+------------+------------+------------+ | :-| | xml_etree_generate| 1.095% | +0.355% | +6.036% | +7.572% | +-----+------------------------+--------+------------+------------+------------+ | :-| | xml_etree_iterparse| 2.324% | -0.452% | +2.155% | +5.810% | +-----+------------------------+--------+------------+------------+------------+ | :-| | xml_etree_parse| 1.309% | -0.115% | -5.283% | +11.921% | +-----+------------------------+--------+------------+------------+------------+ | :-| | xml_etree_process| 1.387% | -0.271% | +6.329% | +8.285% | +-----+------------------------+--------+------------+------------+------------+ * Relative Standard Deviation (Standard Deviation/Average) If this is not displayed properly please visit our results page here: http://languagesperformance.intel.com/65-flat-results-for-python-master-bra… Our lab does a nightly source pull and build of the Python project and measures performance changes against the previous stable version and the previous nightly measurement. This is provided as a service to the community so that quality issues with current hardware can be identified quickly. Intel technologies' features and benefits depend on system configuration and may require enabled hardware, software or service activation. Performance varies depending on system configuration.

1 0

Fix a minor grammar issue in the logging cookbook (GH-3136)
by Brett Cannon 18 Aug '17

18 Aug '17

https://github.com/python/cpython/commit/a3110a0133db6d2a5308700d68f57a7bbc… commit: a3110a0133db6d2a5308700d68f57a7bbcfd4c30 branch: master author: Brett Cannon <brettcannon(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2017-08-18T10:00:31-07:00 summary: Fix a minor grammar issue in the logging cookbook (GH-3136) files: M Doc/howto/logging-cookbook.rst diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 6498ea56957..24cf76b8ba4 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -1055,7 +1055,7 @@ to indicate additional contextual information to be added to the log). So you cannot directly make logging calls using :meth:`str.format` or :class:`string.Template` syntax, because internally the logging package uses %-formatting to merge the format string and the variable arguments. -There would no changing this while preserving backward compatibility, since +There would be no changing this while preserving backward compatibility, since all logging calls which are out there in existing code will be using %-format strings.

1 0

bpo-31231: Fix pythoninfo in Travis config (#3134)
by Victor Stinner 18 Aug '17

18 Aug '17

https://github.com/python/cpython/commit/92b1f90143286385c0ff5be98d3721b905… commit: 92b1f90143286385c0ff5be98d3721b90580a912 branch: master author: Victor Stinner <victor.stinner(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-08-18T17:30:51+02:00 summary: bpo-31231: Fix pythoninfo in Travis config (#3134) bpo-31231, bpo-30871: Replace "./python -m test.pythoninfo" with "make pythoninfo", since macOS uses ./python.exe. files: M .travis.yml diff --git a/.travis.yml b/.travis.yml index 4c431241680..f6e9057e28d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -88,7 +88,7 @@ before_script: echo "$changes" exit 1 fi - ./python -m test.pythoninfo + make pythoninfo script: # Using the built Python as patchcheck.py is built around the idea of using

1 0

Fix broken `Show Source` links on documentation pages (GH-3113) (GH-3128)
by Mariatta 18 Aug '17

18 Aug '17

https://github.com/python/cpython/commit/2281c041b24308811f781935a6c6848d55… commit: 2281c041b24308811f781935a6c6848d55f24311 branch: 2.7 author: Mariatta <Mariatta(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2017-08-18T06:22:32-07:00 summary: Fix broken `Show Source` links on documentation pages (GH-3113) (GH-3128) The `Show Source` was broken because of a change made in sphinx 1.5.1 In Sphinx 1.4.9, the sourcename was "index.txt". In Sphinx 1.5.1+, it is now "index.rst.txt". (cherry picked from commit b9ff498793611d1c6a9b99df464812931a1e2d69) files: M Doc/tools/templates/layout.html diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index cc16ae446ff..8d6d3e5bd11 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -46,7 +46,7 @@ <h3>{{ _('This Page') }}</h3> <ul class="this-page-menu"> <li><a href="{{ pathto('bugs') }}">{% trans %}Report a Bug{% endtrans %}</a></li> - <li><a href="https://github.com/python/cpython/blob/{{ version }}/Doc/{{ sourcename|replace('txt', 'rst') }}" + <li><a href="https://github.com/python/cpython/blob/{{ version }}/Doc/{{ sourcename|replace('.rst.txt', '.rst') }}" rel="nofollow">{% trans %}Show Source{% endtrans %}</a> </li> </ul>

1 0

Fix broken `Show Source` links on documentation pages (GH-3113) (GH-3125)
by Mariatta 18 Aug '17

18 Aug '17

https://github.com/python/cpython/commit/ff64a24c67fcd20bef9305db306b502699… commit: ff64a24c67fcd20bef9305db306b5026990d412b branch: 3.6 author: Mariatta <Mariatta(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2017-08-18T06:21:33-07:00 summary: Fix broken `Show Source` links on documentation pages (GH-3113) (GH-3125) The `Show Source` was broken because of a change made in sphinx 1.5.1 In Sphinx 1.4.9, the sourcename was "index.txt". In Sphinx 1.5.1+, it is now "index.rst.txt". (cherry picked from commit b9ff498793611d1c6a9b99df464812931a1e2d69) files: M Doc/tools/templates/customsourcelink.html diff --git a/Doc/tools/templates/customsourcelink.html b/Doc/tools/templates/customsourcelink.html index 71d0bba6830..21af621efce 100644 --- a/Doc/tools/templates/customsourcelink.html +++ b/Doc/tools/templates/customsourcelink.html @@ -4,7 +4,7 @@ <ul class="this-page-menu"> <li><a href="{{ pathto('bugs') }}">{% trans %}Report a Bug{% endtrans %}</a></li> <li> - <a href="https://github.com/python/cpython/blob/{{ version }}/Doc/{{ sourcename|replace('txt', 'rst') }}" + <a href="https://github.com/python/cpython/blob/{{ version }}/Doc/{{ sourcename|replace('.rst.txt', '.rst') }}" rel="nofollow">{{ _('Show Source') }} </a> </li>

1 0