Mailman 3 December 2017 - Python-checkins

[1 up, 64 flat] Results for Python (master branch) 2017-12-01
by lp_benchmark_robot＠intel.com Dec. 1, 2017

Dec. 1, 2017

Results for project python/master, build date: 2017-12-01 03:02:57-08:00. - commit: bf2b65e - previous commit: 5e3806f - revision date: 2017-12-01 08:08:49+01:00 - environment: Broadwell-EP - cpu: Intel(R) Xeon(R) CPU E5-2699 v4 @ 2.20GHz 2x22 cores, stepping 1, LLC 55 MB - mem: 128 GB - os: Ubuntu 16.04.2 LTS - kernel: 4.4.0-62-generic x86_64 GNU/Linux Baseline results were generated using release v3.6.0, with hash 5c4568a from 2016-12-22 23:38:47+00:00. +-----+------------------------+--------+------------+------------+------------+ | | |relative|change since|change since|current rev | | | benchmark|std_dev*| last run | baseline |run with PGO| +-----+------------------------+--------+------------+------------+------------+ | :-| | 2to3| 0.891% | +0.581% | +5.653% | +6.981% | +-----+------------------------+--------+------------+------------+------------+ | :-| | call_method| 0.915% | +1.892% | +22.959% | +5.594% | +-----+------------------------+--------+------------+------------+------------+ | :-| | call_method_slots| 1.994% | +1.916% | +23.455% | +5.097% | +-----+------------------------+--------+------------+------------+------------+ | :-| | call_method_unknown| 1.604% | +1.094% | +21.572% | +5.007% | +-----+------------------------+--------+------------+------------+------------+ | :-| | call_simple| 2.368% | +3.957% | +9.200% | +7.669% | +-----+------------------------+--------+------------+------------+------------+ | :-| | chameleon| 1.709% | +1.565% | +9.744% | +12.317% | +-----+------------------------+--------+------------+------------+------------+ | :-| | chaos| 0.708% | +1.045% | +7.037% | +9.537% | +-----+------------------------+--------+------------+------------+------------+ | :-| | crypto_pyaes| 0.471% | -0.367% | +0.651% | +8.398% | +-----+------------------------+--------+------------+------------+------------+ | :-| | deltablue| 3.695% | -0.099% | +5.943% | +19.532% | +-----+------------------------+--------+------------+------------+------------+ | :-| | django_template| 3.685% | -3.256% | +11.837% | +4.974% | +-----+------------------------+--------+------------+------------+------------+ | :-| | dulwich_log| 1.065% | -0.927% | +4.326% | +6.839% | +-----+------------------------+--------+------------+------------+------------+ | :-| | fannkuch| 0.373% | -0.152% | +5.622% | +5.118% | +-----+------------------------+--------+------------+------------+------------+ | :-| | float| 0.757% | +1.209% | +3.193% | +5.480% | +-----+------------------------+--------+------------+------------+------------+ | :-| | genshi_text| 0.907% | +0.636% | +14.252% | +8.386% | +-----+------------------------+--------+------------+------------+------------+ | :-| | genshi_xml| 2.254% | +0.554% | +9.232% | +9.210% | +-----+------------------------+--------+------------+------------+------------+ | :-| | go| 0.893% | +0.241% | +5.930% | +10.218% | +-----+------------------------+--------+------------+------------+------------+ | :-| | hexiom| 0.380% | -0.029% | +10.810% | +11.159% | +-----+------------------------+--------+------------+------------+------------+ | :-| | html5lib| 2.927% | -0.536% | +9.017% | +7.717% | +-----+------------------------+--------+------------+------------+------------+ | :-| | json_dumps| 0.989% | +0.447% | +3.310% | +9.878% | +-----+------------------------+--------+------------+------------+------------+ | :-| | json_loads| 1.797% | +1.125% | +0.639% | +12.282% | +-----+------------------------+--------+------------+------------+------------+ | :-| | logging_format| 1.693% | -1.284% | +6.265% | +13.124% | +-----+------------------------+--------+------------+------------+------------+ | :-| | logging_silent| 2.692% | -0.748% | +47.186% | +10.589% | +-----+------------------------+--------+------------+------------+------------+ | :-| | logging_simple| 1.767% | +0.115% | +7.573% | +12.896% | +-----+------------------------+--------+------------+------------+------------+ | :-| | mako| 0.386% | +0.303% | +17.260% | +12.289% | +-----+------------------------+--------+------------+------------+------------+ | :-| | mdp| 2.975% | -0.129% | +7.520% | +11.324% | +-----+------------------------+--------+------------+------------+------------+ | :-| | meteor_contest| 1.376% | +0.136% | +3.353% | +6.336% | +-----+------------------------+--------+------------+------------+------------+ | :-| | nbody| 0.416% | -0.359% | +0.230% | -1.284% | +-----+------------------------+--------+------------+------------+------------+ | :-| | nqueens| 0.777% | +0.935% | +4.760% | +7.175% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pathlib| 1.713% | +0.578% | +5.661% | +10.067% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pickle| 1.146% | -0.962% | +0.866% | +20.690% | +-----+------------------------+--------+------------+------------+------------+ | :-) | pickle_dict| 1.041% | +4.205% | +1.345% | +22.494% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pickle_list| 0.776% | +1.027% | +2.598% | +19.845% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pickle_pure_python| 2.571% | -0.089% | +11.996% | +9.240% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pidigits| 0.069% | +0.062% | +0.159% | +10.235% | +-----+------------------------+--------+------------+------------+------------+ | :-| | python_startup| 0.137% | -0.287% | +10.559% | +5.074% | +-----+------------------------+--------+------------+------------+------------+ | :-| | python_startup_no_site| 0.107% | -0.220% | +1.925% | +5.277% | +-----+------------------------+--------+------------+------------+------------+ | :-| | raytrace| 1.061% | -0.281% | +9.840% | +12.953% | +-----+------------------------+--------+------------+------------+------------+ | :-| | regex_compile| 5.492% | -0.453% | +0.675% | +11.474% | +-----+------------------------+--------+------------+------------+------------+ | :-| | regex_dna| 0.324% | -0.004% | -1.327% | +10.285% | +-----+------------------------+--------+------------+------------+------------+ | :-| | regex_effbot| 2.698% | +3.638% | -2.773% | +7.394% | +-----+------------------------+--------+------------+------------+------------+ | :-| | regex_v8| 1.412% | +0.318% | +5.291% | +6.450% | +-----+------------------------+--------+------------+------------+------------+ | :-| | richards| 1.608% | +0.462% | +8.205% | +13.309% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_fft| 0.745% | -0.311% | -1.979% | +5.249% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_lu| 1.185% | +0.565% | +28.236% | +7.653% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_monte_carlo| 2.630% | +0.290% | +5.294% | +4.261% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_sor| 1.039% | +0.421% | +14.024% | +8.597% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_sparse_mat_mult| 0.555% | -1.063% | -2.170% | +3.558% | +-----+------------------------+--------+------------+------------+------------+ | :-| | spectral_norm| 0.657% | -0.421% | +4.125% | +4.148% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sqlalchemy_declarative| 1.145% | -0.304% | +6.299% | +7.017% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sqlalchemy_imperative| 3.422% | -0.379% | +6.337% | +3.245% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sqlite_synth| 4.592% | +0.931% | +18.148% | +8.487% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sympy_expand| 2.645% | -0.438% | +11.496% | +7.368% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sympy_integrate| 2.035% | -0.253% | +9.021% | +6.761% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sympy_str| 4.153% | -0.391% | +11.137% | +8.440% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sympy_sum| 4.756% | +0.716% | +11.553% | +7.477% | +-----+------------------------+--------+------------+------------+------------+ | :-| | telco| 2.836% | +2.358% | +23.799% | +7.958% | +-----+------------------------+--------+------------+------------+------------+ | :-| | tornado_http| 1.221% | -0.631% | +5.356% | +6.712% | +-----+------------------------+--------+------------+------------+------------+ | :-| | unpack_sequence| 0.844% | +0.192% | +2.283% | +2.423% | +-----+------------------------+--------+------------+------------+------------+ | :-| | unpickle| 5.501% | -1.049% | +8.532% | +19.352% | +-----+------------------------+--------+------------+------------+------------+ | :-| | unpickle_list| 1.394% | -3.145% | -3.027% | +18.572% | +-----+------------------------+--------+------------+------------+------------+ | :-| | unpickle_pure_python| 0.625% | +1.006% | +6.962% | +7.869% | +-----+------------------------+--------+------------+------------+------------+ | :-| | xml_etree_generate| 0.994% | -0.158% | +5.943% | +8.328% | +-----+------------------------+--------+------------+------------+------------+ | :-| | xml_etree_iterparse| 2.363% | -0.470% | +3.128% | +9.055% | +-----+------------------------+--------+------------+------------+------------+ | :-| | xml_etree_parse| 3.055% | -0.316% | -7.390% | +13.744% | +-----+------------------------+--------+------------+------------+------------+ | :-| | xml_etree_process| 1.257% | +0.035% | +7.128% | +9.051% | +-----+------------------------+--------+------------+------------+------------+ * Relative Standard Deviation (Standard Deviation/Average) If this is not displayed properly please visit our results page here: http://languagesperformance.intel.com/1-up-64-flat-results-for-python-maste… Our lab does a nightly source pull and build of the Python project and measures performance changes against the previous stable version and the previous nightly measurement. This is provided as a service to the community so that quality issues with current hardware can be identified quickly. Intel technologies' features and benefits depend on system configuration and may require enabled hardware, software or service activation. Performance varies depending on system configuration.

1 0

bpo-32030: Fix config_get_program_name() on macOS (#4669)
by Victor Stinner Dec. 1, 2017

Dec. 1, 2017

https://github.com/python/cpython/commit/e23c06e2b03452c9aaf0dae52296c85e57… commit: e23c06e2b03452c9aaf0dae52296c85e572f9bcd branch: master author: Victor Stinner <victor.stinner(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-12-01T21:19:48+01:00 summary: bpo-32030: Fix config_get_program_name() on macOS (#4669) files: M Modules/main.c diff --git a/Modules/main.c b/Modules/main.c index 6c1cf0d306a..6c6c8018fd4 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -894,7 +894,7 @@ config_get_program_name(_PyMainInterpreterConfig *config) return SET_DECODE_ERROR("PYTHONEXECUTABLE environment " "variable", len); } - pymain->config.program_name = buffer; + config->program_name = program_name; } #ifdef WITH_NEXT_FRAMEWORK else { @@ -909,7 +909,7 @@ config_get_program_name(_PyMainInterpreterConfig *config) return SET_DECODE_ERROR("__PYVENV_LAUNCHER__ environment " "variable", len); } - pymain->config.program_name = program_name; + config->program_name = program_name; } } #endif /* WITH_NEXT_FRAMEWORK */

1 0

bpo-32030: Add Python/pathconfig.c (#4668)
by Victor Stinner Dec. 1, 2017

Dec. 1, 2017

https://github.com/python/cpython/commit/0ea395ae964c9cd0f499e2ef0d0030c971… commit: 0ea395ae964c9cd0f499e2ef0d0030c971201220 branch: master author: Victor Stinner <victor.stinner(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-12-01T20:50:58+01:00 summary: bpo-32030: Add Python/pathconfig.c (#4668) * Factorize code from PC/getpathp.c and Modules/getpath.c to remove duplicated code * rename pathconfig_clear() to _PyPathConfig_Clear() * Inline _PyPathConfig_Fini() in pymain_impl() and then remove it, since it's a oneliner files: A Python/pathconfig.c M Include/internal/pystate.h M Include/pylifecycle.h M Makefile.pre.in M Modules/getpath.c M Modules/main.c M PC/getpathp.c M PCbuild/pythoncore.vcxproj M PCbuild/pythoncore.vcxproj.filters M Python/pylifecycle.c diff --git a/Include/internal/pystate.h b/Include/internal/pystate.h index 50ad2fc83a3..9d8531965f1 100644 --- a/Include/internal/pystate.h +++ b/Include/internal/pystate.h @@ -52,6 +52,10 @@ typedef struct { #define _PyPathConfig_INIT {.module_search_path = NULL} +PyAPI_DATA(_PyPathConfig) _Py_path_config; + +PyAPI_FUNC(void) _PyPathConfig_Clear(_PyPathConfig *config); + /* Full Python runtime state */ diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index 3db88326aee..fa751692a66 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -105,7 +105,6 @@ PyAPI_FUNC(wchar_t *) Py_GetPath(void); #ifdef Py_BUILD_CORE PyAPI_FUNC(_PyInitError) _PyPathConfig_Init( const _PyMainInterpreterConfig *main_config); -PyAPI_FUNC(void) _PyPathConfig_Fini(void); #endif PyAPI_FUNC(void) Py_SetPath(const wchar_t *); #ifdef MS_WINDOWS diff --git a/Makefile.pre.in b/Makefile.pre.in index f425a89173a..14f6f8abc54 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -337,8 +337,9 @@ PYTHON_OBJS= \ Python/importdl.o \ Python/marshal.o \ Python/modsupport.o \ - Python/mystrtoul.o \ Python/mysnprintf.o \ + Python/mystrtoul.o \ + Python/pathconfig.o \ Python/peephole.o \ Python/pyarena.o \ Python/pyctype.o \ diff --git a/Modules/getpath.c b/Modules/getpath.c index 235badab230..8554dbe2a2c 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -132,7 +132,6 @@ typedef struct { static const wchar_t delimiter[2] = {DELIM, '\0'}; static const wchar_t separator[2] = {SEP, '\0'}; -static _PyPathConfig _Py_path_config = _PyPathConfig_INIT; /* Get file status. Encode the path to the locale encoding. */ @@ -1009,23 +1008,6 @@ calculate_path_impl(const _PyMainInterpreterConfig *main_config, } -static void -pathconfig_clear(_PyPathConfig *config) -{ -#define CLEAR(ATTR) \ - do { \ - PyMem_RawFree(ATTR); \ - ATTR = NULL; \ - } while (0) - - CLEAR(config->prefix); - CLEAR(config->exec_prefix); - CLEAR(config->program_full_path); - CLEAR(config->module_search_path); -#undef CLEAR -} - - /* Initialize paths for Py_GetPath(), Py_GetPrefix(), Py_GetExecPrefix() and Py_GetProgramFullPath() */ _PyInitError @@ -1049,7 +1031,7 @@ _PyPathConfig_Init(const _PyMainInterpreterConfig *main_config) err = calculate_path_impl(main_config, &calculate, &new_path_config); if (_Py_INIT_FAILED(err)) { - pathconfig_clear(&new_path_config); + _PyPathConfig_Clear(&new_path_config); goto done; } @@ -1061,100 +1043,6 @@ _PyPathConfig_Init(const _PyMainInterpreterConfig *main_config) return err; } - -static void -pathconfig_global_init(void) -{ - if (_Py_path_config.module_search_path) { - /* Already initialized */ - return; - } - - _PyInitError err; - _PyMainInterpreterConfig config = _PyMainInterpreterConfig_INIT; - - err = _PyMainInterpreterConfig_ReadEnv(&config); - if (_Py_INIT_FAILED(err)) { - goto error; - } - - err = _PyMainInterpreterConfig_Read(&config); - if (_Py_INIT_FAILED(err)) { - goto error; - } - - err = _PyPathConfig_Init(&config); - if (_Py_INIT_FAILED(err)) { - goto error; - } - - _PyMainInterpreterConfig_Clear(&config); - return; - -error: - _PyMainInterpreterConfig_Clear(&config); - _Py_FatalInitError(err); -} - - -void -_PyPathConfig_Fini(void) -{ - pathconfig_clear(&_Py_path_config); -} - - -/* External interface */ -void -Py_SetPath(const wchar_t *path) -{ - if (path == NULL) { - pathconfig_clear(&_Py_path_config); - return; - } - - _PyPathConfig new_config; - new_config.program_full_path = _PyMem_RawWcsdup(Py_GetProgramName()); - new_config.exec_prefix = _PyMem_RawWcsdup(L""); - new_config.prefix = _PyMem_RawWcsdup(L""); - new_config.module_search_path = _PyMem_RawWcsdup(path); - - pathconfig_clear(&_Py_path_config); - _Py_path_config = new_config; -} - - -wchar_t * -Py_GetPath(void) -{ - pathconfig_global_init(); - return _Py_path_config.module_search_path; -} - - -wchar_t * -Py_GetPrefix(void) -{ - pathconfig_global_init(); - return _Py_path_config.prefix; -} - - -wchar_t * -Py_GetExecPrefix(void) -{ - pathconfig_global_init(); - return _Py_path_config.exec_prefix; -} - - -wchar_t * -Py_GetProgramFullPath(void) -{ - pathconfig_global_init(); - return _Py_path_config.program_full_path; -} - #ifdef __cplusplus } #endif diff --git a/Modules/main.c b/Modules/main.c index 4659c5d670b..6c1cf0d306a 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -1665,12 +1665,12 @@ pymain_impl(_PyMain *pymain) pymain->status = 120; } - /* _PyPathConfig_Fini() cannot be called in Py_FinalizeEx(). + /* _PyPathConfig_Clear() cannot be called in Py_FinalizeEx(). Py_Initialize() and Py_Finalize() can be called multiple times, but it must not "forget" parameters set by Py_SetProgramName(), Py_SetPath() or - Py_SetPythonHome(), whereas _PyPathConfig_Fini() clear all these + Py_SetPythonHome(), whereas _PyPathConfig_Clear() clear all these parameters. */ - _PyPathConfig_Fini(); + _PyPathConfig_Clear(&_Py_path_config); return 0; } diff --git a/PC/getpathp.c b/PC/getpathp.c index f8cfd7e6978..3a0ebc10636 100644 --- a/PC/getpathp.c +++ b/PC/getpathp.c @@ -130,9 +130,6 @@ typedef struct { } PyCalculatePath; -static _PyPathConfig _Py_path_config = _PyPathConfig_INIT; - - /* determine if "ch" is a separator character */ static int is_sep(wchar_t ch) @@ -1061,23 +1058,6 @@ calculate_free(PyCalculatePath *calculate) } -static void -pathconfig_clear(_PyPathConfig *config) -{ -#define CLEAR(ATTR) \ - do { \ - PyMem_RawFree(ATTR); \ - ATTR = NULL; \ - } while (0) - - CLEAR(config->prefix); - CLEAR(config->program_full_path); - CLEAR(config->dll_path); - CLEAR(config->module_search_path); -#undef CLEAR -} - - /* Initialize paths for Py_GetPath(), Py_GetPrefix(), Py_GetExecPrefix() and Py_GetProgramFullPath() */ _PyInitError @@ -1108,110 +1088,13 @@ _PyPathConfig_Init(const _PyMainInterpreterConfig *main_config) done: if (_Py_INIT_FAILED(err)) { - pathconfig_clear(&new_path_config); + _PyPathConfig_Clear(&new_path_config); } calculate_free(&calculate); return err; } -static void -pathconfig_global_init(void) -{ - if (_Py_path_config.module_search_path) { - /* Already initialized */ - return; - } - - _PyInitError err; - _PyMainInterpreterConfig config = _PyMainInterpreterConfig_INIT; - - err = _PyMainInterpreterConfig_ReadEnv(&config); - if (_Py_INIT_FAILED(err)) { - goto error; - } - - err = _PyMainInterpreterConfig_Read(&config); - if (_Py_INIT_FAILED(err)) { - goto error; - } - - err = _PyPathConfig_Init(&config); - if (_Py_INIT_FAILED(err)) { - goto error; - } - - _PyMainInterpreterConfig_Clear(&config); - return; - -error: - _PyMainInterpreterConfig_Clear(&config); - _Py_FatalInitError(err); -} - - -void -_PyPathConfig_Fini(void) -{ - pathconfig_clear(&_Py_path_config); -} - - -/* External interface */ - -void -Py_SetPath(const wchar_t *path) -{ - if (_Py_path_config.module_search_path != NULL) { - pathconfig_clear(&_Py_path_config); - } - - if (path == NULL) { - return; - } - - _PyPathConfig new_config; - new_config.program_full_path = _PyMem_RawWcsdup(Py_GetProgramName()); - new_config.prefix = _PyMem_RawWcsdup(L""); - new_config.dll_path = _PyMem_RawWcsdup(L""); - new_config.module_search_path = _PyMem_RawWcsdup(path); - - pathconfig_clear(&_Py_path_config); - _Py_path_config = new_config; -} - - -wchar_t * -Py_GetPath(void) -{ - pathconfig_global_init(); - return _Py_path_config.module_search_path; -} - - -wchar_t * -Py_GetPrefix(void) -{ - pathconfig_global_init(); - return _Py_path_config.prefix; -} - - -wchar_t * -Py_GetExecPrefix(void) -{ - return Py_GetPrefix(); -} - - -wchar_t * -Py_GetProgramFullPath(void) -{ - pathconfig_global_init(); - return _Py_path_config.program_full_path; -} - - /* Load python3.dll before loading any extension module that might refer to it. That way, we can be sure that always the python3.dll corresponding to this python DLL is loaded, not a python3.dll that might be on the path diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 3793cbda882..b430e05b629 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -381,6 +381,7 @@ <ClCompile Include="..\Python\modsupport.c" /> <ClCompile Include="..\Python\mysnprintf.c" /> <ClCompile Include="..\Python\mystrtoul.c" /> + <ClCompile Include="..\Python\pathconfig.c" /> <ClCompile Include="..\Python\peephole.c" /> <ClCompile Include="..\Python\pyarena.c" /> <ClCompile Include="..\Python\pyctype.c" /> diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 1d33c6e2cc2..c9aa3da355e 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -896,6 +896,9 @@ <ClCompile Include="..\Python\mystrtoul.c"> <Filter>Python</Filter> </ClCompile> + <ClCompile Include="..\Python\pathconfig.c"> + <Filter>Python</Filter> + </ClCompile> <ClCompile Include="..\Python\peephole.c"> <Filter>Python</Filter> </ClCompile> diff --git a/Python/pathconfig.c b/Python/pathconfig.c new file mode 100644 index 00000000000..daa22270369 --- /dev/null +++ b/Python/pathconfig.c @@ -0,0 +1,189 @@ +/* Path configuration like module_search_path (sys.path) */ + +#include "Python.h" +#include "osdefs.h" +#include "internal/pystate.h" + +#ifdef __cplusplus +extern "C" { +#endif + + +_PyPathConfig _Py_path_config = _PyPathConfig_INIT; +#ifdef MS_WINDOWS +static wchar_t *progname = L"python"; +#else +static wchar_t *progname = L"python3"; +#endif +static wchar_t *default_home = NULL; + + +void +_PyPathConfig_Clear(_PyPathConfig *config) +{ +#define CLEAR(ATTR) \ + do { \ + PyMem_RawFree(ATTR); \ + ATTR = NULL; \ + } while (0) + + CLEAR(config->prefix); + CLEAR(config->program_full_path); +#ifdef MS_WINDOWS + CLEAR(config->dll_path); +#else + CLEAR(config->exec_prefix); +#endif + CLEAR(config->module_search_path); +#undef CLEAR +} + + +void +Py_SetProgramName(wchar_t *pn) +{ + if (pn && *pn) + progname = pn; +} + + +wchar_t * +Py_GetProgramName(void) +{ + return progname; +} + + +void +Py_SetPythonHome(wchar_t *home) +{ + default_home = home; +} + + +wchar_t* +Py_GetPythonHome(void) +{ + /* Use a static buffer to avoid heap memory allocation failure. + Py_GetPythonHome() doesn't allow to report error, and the caller + doesn't release memory. */ + static wchar_t buffer[MAXPATHLEN+1]; + + if (default_home) { + return default_home; + } + + char *home = Py_GETENV("PYTHONHOME"); + if (!home) { + return NULL; + } + + size_t size = Py_ARRAY_LENGTH(buffer); + size_t r = mbstowcs(buffer, home, size); + if (r == (size_t)-1 || r >= size) { + /* conversion failed or the static buffer is too small */ + return NULL; + } + + return buffer; +} + + +static void +pathconfig_global_init(void) +{ + if (_Py_path_config.module_search_path) { + /* Already initialized */ + return; + } + + _PyInitError err; + _PyMainInterpreterConfig config = _PyMainInterpreterConfig_INIT; + + err = _PyMainInterpreterConfig_ReadEnv(&config); + if (_Py_INIT_FAILED(err)) { + goto error; + } + + err = _PyMainInterpreterConfig_Read(&config); + if (_Py_INIT_FAILED(err)) { + goto error; + } + + err = _PyPathConfig_Init(&config); + if (_Py_INIT_FAILED(err)) { + goto error; + } + + _PyMainInterpreterConfig_Clear(&config); + return; + +error: + _PyMainInterpreterConfig_Clear(&config); + _Py_FatalInitError(err); +} + + +/* External interface */ + +void +Py_SetPath(const wchar_t *path) +{ + if (path == NULL) { + _PyPathConfig_Clear(&_Py_path_config); + return; + } + + _PyPathConfig new_config; + new_config.program_full_path = _PyMem_RawWcsdup(Py_GetProgramName()); + new_config.prefix = _PyMem_RawWcsdup(L""); +#ifdef MS_WINDOWS + new_config.dll_path = _PyMem_RawWcsdup(L""); +#else + new_config.exec_prefix = _PyMem_RawWcsdup(L""); +#endif + new_config.module_search_path = _PyMem_RawWcsdup(path); + + _PyPathConfig_Clear(&_Py_path_config); + _Py_path_config = new_config; +} + + +wchar_t * +Py_GetPath(void) +{ + pathconfig_global_init(); + return _Py_path_config.module_search_path; +} + + +wchar_t * +Py_GetPrefix(void) +{ + pathconfig_global_init(); + return _Py_path_config.prefix; +} + + +wchar_t * +Py_GetExecPrefix(void) +{ +#ifdef MS_WINDOWS + return Py_GetPrefix(); +#else + pathconfig_global_init(); + return _Py_path_config.exec_prefix; +#endif +} + + +wchar_t * +Py_GetProgramFullPath(void) +{ + pathconfig_global_init(); + return _Py_path_config.program_full_path; +} + +#ifdef __cplusplus +} +#endif diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index a1b29f2a0eb..8d71154c563 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1489,61 +1489,6 @@ Py_EndInterpreter(PyThreadState *tstate) PyInterpreterState_Delete(interp); } -#ifdef MS_WINDOWS -static wchar_t *progname = L"python"; -#else -static wchar_t *progname = L"python3"; -#endif - -void -Py_SetProgramName(wchar_t *pn) -{ - if (pn && *pn) - progname = pn; -} - -wchar_t * -Py_GetProgramName(void) -{ - return progname; -} - -static wchar_t *default_home = NULL; - -void -Py_SetPythonHome(wchar_t *home) -{ - default_home = home; -} - - -wchar_t* -Py_GetPythonHome(void) -{ - /* Use a static buffer to avoid heap memory allocation failure. - Py_GetPythonHome() doesn't allow to report error, and the caller - doesn't release memory. */ - static wchar_t buffer[MAXPATHLEN+1]; - - if (default_home) { - return default_home; - } - - char *home = Py_GETENV("PYTHONHOME"); - if (!home) { - return NULL; - } - - size_t size = Py_ARRAY_LENGTH(buffer); - size_t r = mbstowcs(buffer, home, size); - if (r == (size_t)-1 || r >= size) { - /* conversion failed or the static buffer is too small */ - return NULL; - } - - return buffer; -} - /* Add the __main__ module */ static _PyInitError

1 0

bpo-32030: Don't call _PyPathConfig_Fini() in Py_FinalizeEx() (#4667)
by Victor Stinner Dec. 1, 2017

Dec. 1, 2017

https://github.com/python/cpython/commit/ebac19dad6263141d5db0a2c923efe049d… commit: ebac19dad6263141d5db0a2c923efe049dba99d2 branch: master author: Victor Stinner <victor.stinner(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-12-01T20:09:52+01:00 summary: bpo-32030: Don't call _PyPathConfig_Fini() in Py_FinalizeEx() (#4667) Changes: * _PyPathConfig_Fini() cannot be called in Py_FinalizeEx(). Py_Initialize() and Py_Finalize() can be called multiple times, but it must not "forget" parameters set by Py_SetProgramName(), Py_SetPath() or Py_SetPythonHome(), whereas _PyPathConfig_Fini() clear all these parameters. * config_get_program_name() and calculate_program_full_path() now also decode paths using Py_DecodeLocale() to use the surrogateescape error handler, rather than decoding using mbstowcs() which is strict. * Change _Py_CheckPython3() prototype: () => (void) * Truncate a few lines which were too long files: M Include/pylifecycle.h M Modules/getpath.c M Modules/main.c M PC/getpathp.c M Python/pylifecycle.c diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index d32c98b6985..3db88326aee 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -109,7 +109,7 @@ PyAPI_FUNC(void) _PyPathConfig_Fini(void); #endif PyAPI_FUNC(void) Py_SetPath(const wchar_t *); #ifdef MS_WINDOWS -int _Py_CheckPython3(); +int _Py_CheckPython3(void); #endif /* In their own files */ diff --git a/Modules/getpath.c b/Modules/getpath.c index 183717d8171..235badab230 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -625,11 +625,13 @@ calculate_program_full_path(const _PyMainInterpreterConfig *main_config, else if(0 == _NSGetExecutablePath(execpath, &nsexeclength) && execpath[0] == SEP) { - size_t r = mbstowcs(program_full_path, execpath, MAXPATHLEN+1); - if (r == (size_t)-1 || r > MAXPATHLEN) { - /* Could not convert execpath, or it's too long. */ - program_full_path[0] = '\0'; + size_t len; + wchar_t *path = Py_DecodeLocale(execpath, &len); + if (path == NULL) { + return DECODE_LOCALE_ERR("executable path", len); } + wcsncpy(program_full_path, path, MAXPATHLEN); + PyMem_RawFree(path); } #endif /* __APPLE__ */ else if (calculate->path_env) { diff --git a/Modules/main.c b/Modules/main.c index caec97f8642..4659c5d670b 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -888,15 +888,12 @@ config_get_program_name(_PyMainInterpreterConfig *config) See Lib/plat-mac/bundlebuiler.py for details about the bootstrap script. */ if ((p = Py_GETENV("PYTHONEXECUTABLE")) && *p != '\0') { - wchar_t* buffer; - size_t len = strlen(p) + 1; - - buffer = PyMem_RawMalloc(len * sizeof(wchar_t)); - if (buffer == NULL) { - return _Py_INIT_NO_MEMORY(); + size_t len; + wchar_t* program_name = Py_DecodeLocale(p, &len); + if (program_name == NULL) { + return SET_DECODE_ERROR("PYTHONEXECUTABLE environment " + "variable", len); } - - mbstowcs(buffer, p, len); pymain->config.program_name = buffer; } #ifdef WITH_NEXT_FRAMEWORK @@ -907,11 +904,12 @@ config_get_program_name(_PyMainInterpreterConfig *config) * the argv0 of the stub executable */ size_t len; - wchar_t* wbuf = Py_DecodeLocale(pyvenv_launcher, &len); - if (wbuf == NULL) { - return SET_DECODE_ERROR("__PYVENV_LAUNCHER__", len); + wchar_t* program_name = Py_DecodeLocale(pyvenv_launcher, &len); + if (program_name == NULL) { + return SET_DECODE_ERROR("__PYVENV_LAUNCHER__ environment " + "variable", len); } - pymain->config.program_name = wbuf; + pymain->config.program_name = program_name; } } #endif /* WITH_NEXT_FRAMEWORK */ @@ -1666,6 +1664,14 @@ pymain_impl(_PyMain *pymain) other special meaning */ pymain->status = 120; } + + /* _PyPathConfig_Fini() cannot be called in Py_FinalizeEx(). + Py_Initialize() and Py_Finalize() can be called multiple times, but it + must not "forget" parameters set by Py_SetProgramName(), Py_SetPath() or + Py_SetPythonHome(), whereas _PyPathConfig_Fini() clear all these + parameters. */ + _PyPathConfig_Fini(); + return 0; } diff --git a/PC/getpathp.c b/PC/getpathp.c index 89d37a84133..f8cfd7e6978 100644 --- a/PC/getpathp.c +++ b/PC/getpathp.c @@ -721,12 +721,16 @@ static int get_pth_filename(wchar_t *spbuffer, _PyPathConfig *config) { if (config->dll_path[0]) { - if (!change_ext(spbuffer, config->dll_path, L"._pth") && exists(spbuffer)) { + if (!change_ext(spbuffer, config->dll_path, L"._pth") && + exists(spbuffer)) + { return 1; } } if (config->program_full_path[0]) { - if (!change_ext(spbuffer, config->program_full_path, L"._pth") && exists(spbuffer)) { + if (!change_ext(spbuffer, config->program_full_path, L"._pth") && + exists(spbuffer)) + { return 1; } } @@ -823,8 +827,10 @@ calculate_module_search_path(const _PyMainInterpreterConfig *main_config, #endif /* We only use the default relative PYTHONPATH if we haven't anything better to use! */ - int skipdefault = (main_config->module_search_path_env!=NULL || calculate->home!=NULL || \ - calculate->machine_path!=NULL || calculate->user_path!=NULL); + int skipdefault = (main_config->module_search_path_env != NULL || + calculate->home != NULL || + calculate->machine_path != NULL || + calculate->user_path != NULL); /* We need to construct a path from the following parts. (1) the PYTHONPATH environment variable, if set; @@ -882,7 +888,8 @@ calculate_module_search_path(const _PyMainInterpreterConfig *main_config, start_buf = buf; if (main_config->module_search_path_env) { - if (wcscpy_s(buf, bufsz - (buf - start_buf), main_config->module_search_path_env)) { + if (wcscpy_s(buf, bufsz - (buf - start_buf), + main_config->module_search_path_env)) { return INIT_ERR_BUFFER_OVERFLOW(); } buf = wcschr(buf, L'\0'); @@ -1214,7 +1221,7 @@ Py_GetProgramFullPath(void) static int python3_checked = 0; static HANDLE hPython3; int -_Py_CheckPython3() +_Py_CheckPython3(void) { wchar_t py3path[MAXPATHLEN+1]; wchar_t *s; diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index f0a49f91fb8..a1b29f2a0eb 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1273,8 +1273,6 @@ Py_FinalizeEx(void) call_ll_exitfuncs(); - _PyPathConfig_Fini(); - _PyRuntime_Finalize(); return status; }

1 0

bpo-32030: Fix Py_GetPath(): init program_name (#4665)
by Victor Stinner Dec. 1, 2017

Dec. 1, 2017

https://github.com/python/cpython/commit/9ac3d8882712c9675c3d2f9f84af6b5729… commit: 9ac3d8882712c9675c3d2f9f84af6b5729575cde branch: master author: Victor Stinner <victor.stinner(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-12-01T19:30:41+01:00 summary: bpo-32030: Fix Py_GetPath(): init program_name (#4665) * _PyMainInterpreterConfig_ReadEnv() now sets program_name from environment variables and pymain_parse_envvars() implements the falls back on argv[0]. * Remove _PyMain.program_name: use the program_name from _PyMainInterpreterConfig * Move the Py_SetProgramName() call back to pymain_init_python(), just before _Py_InitializeCore(). * pathconfig_global_init() now also calls _PyMainInterpreterConfig_Read() to set program_name if it isn't set yet * Cleanup PyCalculatePath: pass main_config to subfunctions to get directly fields from main_config (home, module_search_path_env and program_name) files: M Modules/getpath.c M Modules/main.c M PC/getpathp.c diff --git a/Modules/getpath.c b/Modules/getpath.c index 9f5e8b3ff5c..183717d8171 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -117,10 +117,7 @@ extern "C" { typedef struct { wchar_t *path_env; /* PATH environment variable */ - wchar_t *home; /* PYTHONHOME environment variable */ - wchar_t *module_search_path_env; /* PYTHONPATH environment variable */ - wchar_t *program_name; /* Program name */ wchar_t *pythonpath; /* PYTHONPATH define */ wchar_t *prefix; /* PREFIX define */ wchar_t *exec_prefix; /* EXEC_PREFIX define */ @@ -360,14 +357,15 @@ find_env_config_value(FILE * env_file, const wchar_t * key, wchar_t * value) bytes long. */ static int -search_for_prefix(PyCalculatePath *calculate, wchar_t *prefix) +search_for_prefix(const _PyMainInterpreterConfig *main_config, + PyCalculatePath *calculate, wchar_t *prefix) { size_t n; wchar_t *vpath; /* If PYTHONHOME is set, we believe it unconditionally */ - if (calculate->home) { - wcsncpy(prefix, calculate->home, MAXPATHLEN); + if (main_config->home) { + wcsncpy(prefix, main_config->home, MAXPATHLEN); prefix[MAXPATHLEN] = L'\0'; wchar_t *delim = wcschr(prefix, DELIM); if (delim) { @@ -426,9 +424,10 @@ search_for_prefix(PyCalculatePath *calculate, wchar_t *prefix) static void -calculate_prefix(PyCalculatePath *calculate, wchar_t *prefix) +calculate_prefix(const _PyMainInterpreterConfig *main_config, + PyCalculatePath *calculate, wchar_t *prefix) { - calculate->prefix_found = search_for_prefix(calculate, prefix); + calculate->prefix_found = search_for_prefix(main_config, calculate, prefix); if (!calculate->prefix_found) { if (!Py_FrozenFlag) { fprintf(stderr, @@ -470,18 +469,19 @@ calculate_reduce_prefix(PyCalculatePath *calculate, wchar_t *prefix) MAXPATHLEN bytes long. */ static int -search_for_exec_prefix(PyCalculatePath *calculate, wchar_t *exec_prefix) +search_for_exec_prefix(const _PyMainInterpreterConfig *main_config, + PyCalculatePath *calculate, wchar_t *exec_prefix) { size_t n; /* If PYTHONHOME is set, we believe it unconditionally */ - if (calculate->home) { - wchar_t *delim = wcschr(calculate->home, DELIM); + if (main_config->home) { + wchar_t *delim = wcschr(main_config->home, DELIM); if (delim) { wcsncpy(exec_prefix, delim+1, MAXPATHLEN); } else { - wcsncpy(exec_prefix, calculate->home, MAXPATHLEN); + wcsncpy(exec_prefix, main_config->home, MAXPATHLEN); } exec_prefix[MAXPATHLEN] = L'\0'; joinpath(exec_prefix, calculate->lib_python); @@ -552,9 +552,12 @@ search_for_exec_prefix(PyCalculatePath *calculate, wchar_t *exec_prefix) static void -calculate_exec_prefix(PyCalculatePath *calculate, wchar_t *exec_prefix) +calculate_exec_prefix(const _PyMainInterpreterConfig *main_config, + PyCalculatePath *calculate, wchar_t *exec_prefix) { - calculate->exec_prefix_found = search_for_exec_prefix(calculate, exec_prefix); + calculate->exec_prefix_found = search_for_exec_prefix(main_config, + calculate, + exec_prefix); if (!calculate->exec_prefix_found) { if (!Py_FrozenFlag) { fprintf(stderr, @@ -585,7 +588,8 @@ calculate_reduce_exec_prefix(PyCalculatePath *calculate, wchar_t *exec_prefix) static _PyInitError -calculate_program_full_path(PyCalculatePath *calculate, _PyPathConfig *config) +calculate_program_full_path(const _PyMainInterpreterConfig *main_config, + PyCalculatePath *calculate, _PyPathConfig *config) { wchar_t program_full_path[MAXPATHLEN+1]; memset(program_full_path, 0, sizeof(program_full_path)); @@ -604,8 +608,8 @@ calculate_program_full_path(PyCalculatePath *calculate, _PyPathConfig *config) * other way to find a directory to start the search from. If * $PATH isn't exported, you lose. */ - if (wcschr(calculate->program_name, SEP)) { - wcsncpy(program_full_path, calculate->program_name, MAXPATHLEN); + if (wcschr(main_config->program_name, SEP)) { + wcsncpy(program_full_path, main_config->program_name, MAXPATHLEN); } #ifdef __APPLE__ /* On Mac OS X, if a script uses an interpreter of the form @@ -645,7 +649,7 @@ calculate_program_full_path(PyCalculatePath *calculate, _PyPathConfig *config) wcsncpy(program_full_path, path, MAXPATHLEN); } - joinpath(program_full_path, calculate->program_name); + joinpath(program_full_path, main_config->program_name); if (isxfile(program_full_path)) { break; } @@ -810,14 +814,15 @@ calculate_zip_path(PyCalculatePath *calculate, const wchar_t *prefix) static _PyInitError -calculate_module_search_path(PyCalculatePath *calculate, +calculate_module_search_path(const _PyMainInterpreterConfig *main_config, + PyCalculatePath *calculate, const wchar_t *prefix, const wchar_t *exec_prefix, _PyPathConfig *config) { /* Calculate size of return buffer */ size_t bufsz = 0; - if (calculate->module_search_path_env != NULL) { - bufsz += wcslen(calculate->module_search_path_env) + 1; + if (main_config->module_search_path_env != NULL) { + bufsz += wcslen(main_config->module_search_path_env) + 1; } wchar_t *defpath = calculate->pythonpath; @@ -851,8 +856,8 @@ calculate_module_search_path(PyCalculatePath *calculate, buf[0] = '\0'; /* Run-time value of $PYTHONPATH goes first */ - if (calculate->module_search_path_env) { - wcscpy(buf, calculate->module_search_path_env); + if (main_config->module_search_path_env) { + wcscpy(buf, main_config->module_search_path_env); wcscat(buf, delimiter); } @@ -903,10 +908,6 @@ static _PyInitError calculate_init(PyCalculatePath *calculate, const _PyMainInterpreterConfig *main_config) { - calculate->home = main_config->home; - calculate->module_search_path_env = main_config->module_search_path_env; - calculate->program_name = main_config->program_name; - size_t len; char *path = getenv("PATH"); if (path) { @@ -948,9 +949,12 @@ calculate_free(PyCalculatePath *calculate) static _PyInitError -calculate_path_impl(PyCalculatePath *calculate, _PyPathConfig *config) +calculate_path_impl(const _PyMainInterpreterConfig *main_config, + PyCalculatePath *calculate, _PyPathConfig *config) { - _PyInitError err = calculate_program_full_path(calculate, config); + _PyInitError err; + + err = calculate_program_full_path(main_config, calculate, config); if (_Py_INIT_FAILED(err)) { return err; } @@ -964,13 +968,13 @@ calculate_path_impl(PyCalculatePath *calculate, _PyPathConfig *config) wchar_t prefix[MAXPATHLEN+1]; memset(prefix, 0, sizeof(prefix)); - calculate_prefix(calculate, prefix); + calculate_prefix(main_config, calculate, prefix); calculate_zip_path(calculate, prefix); wchar_t exec_prefix[MAXPATHLEN+1]; memset(exec_prefix, 0, sizeof(exec_prefix)); - calculate_exec_prefix(calculate, exec_prefix); + calculate_exec_prefix(main_config, calculate, exec_prefix); if ((!calculate->prefix_found || !calculate->exec_prefix_found) && !Py_FrozenFlag) @@ -979,8 +983,8 @@ calculate_path_impl(PyCalculatePath *calculate, _PyPathConfig *config) "Consider setting $PYTHONHOME to <prefix>[:<exec_prefix>]\n"); } - err = calculate_module_search_path(calculate, prefix, exec_prefix, - config); + err = calculate_module_search_path(main_config, calculate, + prefix, exec_prefix, config); if (_Py_INIT_FAILED(err)) { return err; } @@ -1041,7 +1045,7 @@ _PyPathConfig_Init(const _PyMainInterpreterConfig *main_config) _PyPathConfig new_path_config; memset(&new_path_config, 0, sizeof(new_path_config)); - err = calculate_path_impl(&calculate, &new_path_config); + err = calculate_path_impl(main_config, &calculate, &new_path_config); if (_Py_INIT_FAILED(err)) { pathconfig_clear(&new_path_config); goto done; @@ -1068,14 +1072,26 @@ pathconfig_global_init(void) _PyMainInterpreterConfig config = _PyMainInterpreterConfig_INIT; err = _PyMainInterpreterConfig_ReadEnv(&config); - if (!_Py_INIT_FAILED(err)) { - err = _PyPathConfig_Init(&config); + if (_Py_INIT_FAILED(err)) { + goto error; + } + + err = _PyMainInterpreterConfig_Read(&config); + if (_Py_INIT_FAILED(err)) { + goto error; } - _PyMainInterpreterConfig_Clear(&config); + err = _PyPathConfig_Init(&config); if (_Py_INIT_FAILED(err)) { - _Py_FatalInitError(err); + goto error; } + + _PyMainInterpreterConfig_Clear(&config); + return; + +error: + _PyMainInterpreterConfig_Clear(&config); + _Py_FatalInitError(err); } diff --git a/Modules/main.c b/Modules/main.c index e9d524a1463..caec97f8642 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -412,7 +412,6 @@ typedef struct { /* non-zero if filename, command (-c) or module (-m) is set on the command line */ int run_code; - wchar_t *program_name; /* Error message if a function failed */ _PyInitError err; /* PYTHONWARNINGS env var */ @@ -429,7 +428,6 @@ typedef struct { .config = _PyMainInterpreterConfig_INIT, \ .main_importer_path = NULL, \ .run_code = -1, \ - .program_name = NULL, \ .err = _Py_INIT_OK(), \ .env_warning_options = {0, NULL}} @@ -455,7 +453,6 @@ pymain_free_impl(_PyMain *pymain) pymain_optlist_clear(&pymain->env_warning_options); Py_CLEAR(pymain->main_importer_path); - PyMem_RawFree(pymain->program_name); _PyMainInterpreterConfig_Clear(&pymain->config); @@ -874,14 +871,11 @@ pymain_init_stdio(_PyMain *pymain) /* Get the program name: use PYTHONEXECUTABLE and __PYVENV_LAUNCHER__ - environment variables on macOS if available, use argv[0] by default. - - Return 0 on success. - Set pymain->err and return -1 on error. */ -static int -pymain_get_program_name(_PyMain *pymain) + environment variables on macOS if available. */ +static _PyInitError +config_get_program_name(_PyMainInterpreterConfig *config) { - assert(pymain->program_name == NULL); + assert(config->program_name == NULL); #ifdef __APPLE__ char *p; /* On MacOS X, when the Python interpreter is embedded in an @@ -899,12 +893,11 @@ pymain_get_program_name(_PyMain *pymain) buffer = PyMem_RawMalloc(len * sizeof(wchar_t)); if (buffer == NULL) { - pymain->err = _Py_INIT_NO_MEMORY(); - return -1; + return _Py_INIT_NO_MEMORY(); } mbstowcs(buffer, p, len); - pymain->program_name = buffer; + pymain->config.program_name = buffer; } #ifdef WITH_NEXT_FRAMEWORK else { @@ -916,19 +909,26 @@ pymain_get_program_name(_PyMain *pymain) size_t len; wchar_t* wbuf = Py_DecodeLocale(pyvenv_launcher, &len); if (wbuf == NULL) { - SET_DECODE_ERROR("__PYVENV_LAUNCHER__", len); - return -1; + return SET_DECODE_ERROR("__PYVENV_LAUNCHER__", len); } - pymain->program_name = wbuf; + pymain->config.program_name = wbuf; } } #endif /* WITH_NEXT_FRAMEWORK */ #endif /* __APPLE__ */ + return _Py_INIT_OK(); +} - if (pymain->program_name == NULL) { + +/* If config_get_program_name() found no program name: use argv[0] by default. + Return 0 on success. Set pymain->err and return -1 on error. */ +static int +pymain_get_program_name(_PyMain *pymain) +{ + if (pymain->config.program_name == NULL) { /* Use argv[0] by default */ - pymain->program_name = pymain_wstrdup(pymain, pymain->argv[0]); - if (pymain->program_name == NULL) { + pymain->config.program_name = pymain_wstrdup(pymain, pymain->argv[0]); + if (pymain->config.program_name == NULL) { return -1; } } @@ -1451,11 +1451,9 @@ _PyMainInterpreterConfig_ReadEnv(_PyMainInterpreterConfig *config) return err; } - /* FIXME: _PyMainInterpreterConfig_Read() has the same code. Remove it - here? See also pymain_get_program_name() and pymain_parse_envvars(). */ - config->program_name = _PyMem_RawWcsdup(Py_GetProgramName()); - if (config->program_name == NULL) { - return _Py_INIT_NO_MEMORY(); + err = config_get_program_name(config); + if (_Py_INIT_FAILED(err)) { + return err; } return _Py_INIT_OK(); @@ -1481,25 +1479,17 @@ pymain_parse_envvars(_PyMain *pymain) if (pymain_warnings_envvar(pymain) < 0) { return -1; } - if (pymain_get_program_name(pymain) < 0) { - return -1; - } - core_config->allocator = Py_GETENV("PYTHONMALLOC"); - - /* FIXME: move pymain_get_program_name() code into - _PyMainInterpreterConfig_ReadEnv(). - Problem: _PyMainInterpreterConfig_ReadEnv() doesn't have access - to argv[0]. */ - Py_SetProgramName(pymain->program_name); - /* Don't free program_name here: the argument to Py_SetProgramName - must remain valid until Py_FinalizeEx is called. The string is freed - by pymain_free(). */ _PyInitError err = _PyMainInterpreterConfig_ReadEnv(&pymain->config); if (_Py_INIT_FAILED(pymain->err)) { pymain->err = err; return -1; } + if (pymain_get_program_name(pymain) < 0) { + return -1; + } + + core_config->allocator = Py_GETENV("PYTHONMALLOC"); /* -X options */ if (pymain_get_xoption(pymain, L"showrefcount")) { @@ -1578,6 +1568,11 @@ pymain_init_python(_PyMain *pymain) { pymain_init_stdio(pymain); + Py_SetProgramName(pymain->config.program_name); + /* Don't free program_name here: the argument to Py_SetProgramName + must remain valid until Py_FinalizeEx is called. The string is freed + by pymain_free(). */ + pymain->err = _Py_InitializeCore(&pymain->core_config); if (_Py_INIT_FAILED(pymain->err)) { return -1; diff --git a/PC/getpathp.c b/PC/getpathp.c index ad04b6b6efc..89d37a84133 100644 --- a/PC/getpathp.c +++ b/PC/getpathp.c @@ -118,7 +118,6 @@ #endif typedef struct { - wchar_t *module_search_path_env; /* PYTHONPATH environment variable */ wchar_t *path_env; /* PATH environment variable */ wchar_t *home; /* PYTHONHOME environment variable */ @@ -126,7 +125,6 @@ typedef struct { wchar_t *machine_path; /* from HKEY_LOCAL_MACHINE */ wchar_t *user_path; /* from HKEY_CURRENT_USER */ - wchar_t *program_name; /* Program name */ wchar_t argv0_path[MAXPATHLEN+1]; wchar_t zip_path[MAXPATHLEN+1]; } PyCalculatePath; @@ -503,7 +501,8 @@ get_dll_path(PyCalculatePath *calculate, _PyPathConfig *config) static _PyInitError -get_program_full_path(PyCalculatePath *calculate, _PyPathConfig *config) +get_program_full_path(const _PyMainInterpreterConfig *main_config, + PyCalculatePath *calculate, _PyPathConfig *config) { wchar_t program_full_path[MAXPATHLEN+1]; memset(program_full_path, 0, sizeof(program_full_path)); @@ -518,12 +517,13 @@ get_program_full_path(PyCalculatePath *calculate, _PyPathConfig *config) * $PATH isn't exported, you lose. */ #ifdef ALTSEP - if (wcschr(calculate->program_name, SEP) || wcschr(calculate->program_name, ALTSEP)) + if (wcschr(main_config->program_name, SEP) || + wcschr(main_config->program_name, ALTSEP)) #else - if (wcschr(calculate->program_name, SEP)) + if (wcschr(main_config->program_name, SEP)) #endif { - wcsncpy(program_full_path, calculate->program_name, MAXPATHLEN); + wcsncpy(program_full_path, main_config->program_name, MAXPATHLEN); } else if (calculate->path_env) { wchar_t *path = calculate->path_env; @@ -542,7 +542,7 @@ get_program_full_path(PyCalculatePath *calculate, _PyPathConfig *config) } /* join() is safe for MAXPATHLEN+1 size buffer */ - join(program_full_path, calculate->program_name); + join(program_full_path, main_config->program_name); if (exists(program_full_path)) { break; } @@ -713,9 +713,6 @@ calculate_init(PyCalculatePath *calculate, const _PyMainInterpreterConfig *main_config) { calculate->home = main_config->home; - calculate->module_search_path_env = main_config->module_search_path_env; - calculate->program_name = main_config->program_name; - calculate->path_env = _wgetenv(L"PATH"); } @@ -815,7 +812,9 @@ calculate_home_prefix(PyCalculatePath *calculate, wchar_t *prefix) static _PyInitError -calculate_module_search_path(PyCalculatePath *calculate, _PyPathConfig *config, wchar_t *prefix) +calculate_module_search_path(const _PyMainInterpreterConfig *main_config, + PyCalculatePath *calculate, _PyPathConfig *config, + wchar_t *prefix) { int skiphome = calculate->home==NULL ? 0 : 1; #ifdef Py_ENABLE_SHARED @@ -824,7 +823,7 @@ calculate_module_search_path(PyCalculatePath *calculate, _PyPathConfig *config, #endif /* We only use the default relative PYTHONPATH if we haven't anything better to use! */ - int skipdefault = (calculate->module_search_path_env!=NULL || calculate->home!=NULL || \ + int skipdefault = (main_config->module_search_path_env!=NULL || calculate->home!=NULL || \ calculate->machine_path!=NULL || calculate->user_path!=NULL); /* We need to construct a path from the following parts. @@ -861,8 +860,8 @@ calculate_module_search_path(PyCalculatePath *calculate, _PyPathConfig *config, bufsz += wcslen(calculate->machine_path) + 1; } bufsz += wcslen(calculate->zip_path) + 1; - if (calculate->module_search_path_env != NULL) { - bufsz += wcslen(calculate->module_search_path_env) + 1; + if (main_config->module_search_path_env != NULL) { + bufsz += wcslen(main_config->module_search_path_env) + 1; } wchar_t *buf, *start_buf; @@ -870,9 +869,9 @@ calculate_module_search_path(PyCalculatePath *calculate, _PyPathConfig *config, if (buf == NULL) { /* We can't exit, so print a warning and limp along */ fprintf(stderr, "Can't malloc dynamic PYTHONPATH.\n"); - if (calculate->module_search_path_env) { + if (main_config->module_search_path_env) { fprintf(stderr, "Using environment $PYTHONPATH.\n"); - config->module_search_path = calculate->module_search_path_env; + config->module_search_path = main_config->module_search_path_env; } else { fprintf(stderr, "Using default static path.\n"); @@ -882,8 +881,8 @@ calculate_module_search_path(PyCalculatePath *calculate, _PyPathConfig *config, } start_buf = buf; - if (calculate->module_search_path_env) { - if (wcscpy_s(buf, bufsz - (buf - start_buf), calculate->module_search_path_env)) { + if (main_config->module_search_path_env) { + if (wcscpy_s(buf, bufsz - (buf - start_buf), main_config->module_search_path_env)) { return INIT_ERR_BUFFER_OVERFLOW(); } buf = wcschr(buf, L'\0'); @@ -996,8 +995,8 @@ calculate_module_search_path(PyCalculatePath *calculate, _PyPathConfig *config, static _PyInitError -calculate_path_impl(PyCalculatePath *calculate, _PyPathConfig *config, - const _PyMainInterpreterConfig *main_config) +calculate_path_impl(const _PyMainInterpreterConfig *main_config, + PyCalculatePath *calculate, _PyPathConfig *config) { _PyInitError err; @@ -1006,7 +1005,7 @@ calculate_path_impl(PyCalculatePath *calculate, _PyPathConfig *config, return err; } - err = get_program_full_path(calculate, config); + err = get_program_full_path(main_config, calculate, config); if (_Py_INIT_FAILED(err)) { return err; } @@ -1032,7 +1031,7 @@ calculate_path_impl(PyCalculatePath *calculate, _PyPathConfig *config, calculate_home_prefix(calculate, prefix); - err = calculate_module_search_path(calculate, config, prefix); + err = calculate_module_search_path(main_config, calculate, config, prefix); if (_Py_INIT_FAILED(err)) { return err; } @@ -1092,7 +1091,7 @@ _PyPathConfig_Init(const _PyMainInterpreterConfig *main_config) _PyPathConfig new_path_config; memset(&new_path_config, 0, sizeof(new_path_config)); - err = calculate_path_impl(&calculate, &new_path_config, main_config); + err = calculate_path_impl(main_config, &calculate, &new_path_config); if (_Py_INIT_FAILED(err)) { goto done; } @@ -1121,14 +1120,26 @@ pathconfig_global_init(void) _PyMainInterpreterConfig config = _PyMainInterpreterConfig_INIT; err = _PyMainInterpreterConfig_ReadEnv(&config); - if (!_Py_INIT_FAILED(err)) { - err = _PyPathConfig_Init(&config); + if (_Py_INIT_FAILED(err)) { + goto error; + } + + err = _PyMainInterpreterConfig_Read(&config); + if (_Py_INIT_FAILED(err)) { + goto error; } - _PyMainInterpreterConfig_Clear(&config); + err = _PyPathConfig_Init(&config); if (_Py_INIT_FAILED(err)) { - _Py_FatalInitError(err); + goto error; } + + _PyMainInterpreterConfig_Clear(&config); + return; + +error: + _PyMainInterpreterConfig_Clear(&config); + _Py_FatalInitError(err); }

1 0

bpo-32030: Cleanup "path config" code (#4663)
by Victor Stinner Dec. 1, 2017

Dec. 1, 2017

https://github.com/python/cpython/commit/b64de46aae148cfab0980e0ad478da7aaf… commit: b64de46aae148cfab0980e0ad478da7aafc44900 branch: master author: Victor Stinner <victor.stinner(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-12-01T18:27:09+01:00 summary: bpo-32030: Cleanup "path config" code (#4663) * Rename PyPathConfig structure to _PyPathConfig and move it to Include/internal/pystate.h * Rename path_config to _Py_path_config * _PyPathConfig: Rename program_name field to program_full_path * Add assert(str != NULL); to _PyMem_RawWcsdup(), _PyMem_RawStrdup() and _PyMem_Strdup(). * Rename calculate_path() to pathconfig_global_init(). The function now does nothing if it's already initiallized. files: M Include/internal/pystate.h M Modules/getpath.c M Objects/obmalloc.c M PC/getpathp.c diff --git a/Include/internal/pystate.h b/Include/internal/pystate.h index 7056e105ff7..50ad2fc83a3 100644 --- a/Include/internal/pystate.h +++ b/Include/internal/pystate.h @@ -37,6 +37,22 @@ struct _gilstate_runtime_state { #define _PyGILState_check_enabled _PyRuntime.gilstate.check_enabled +typedef struct { + /* Full path to the Python program */ + wchar_t *program_full_path; + wchar_t *prefix; +#ifdef MS_WINDOWS + wchar_t *dll_path; +#else + wchar_t *exec_prefix; +#endif + /* Set by Py_SetPath(), or computed by _PyPathConfig_Init() */ + wchar_t *module_search_path; +} _PyPathConfig; + +#define _PyPathConfig_INIT {.module_search_path = NULL} + + /* Full Python runtime state */ typedef struct pyruntimestate { diff --git a/Modules/getpath.c b/Modules/getpath.c index 02d626cea73..9f5e8b3ff5c 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -1,6 +1,7 @@ /* Return the initial module search path. */ #include "Python.h" +#include "internal/pystate.h" #include "osdefs.h" #include <sys/types.h> @@ -114,14 +115,6 @@ extern "C" { ? _Py_INIT_USER_ERR("cannot decode " #NAME) \ : _Py_INIT_NO_MEMORY() - -typedef struct { - wchar_t *prefix; - wchar_t *exec_prefix; - wchar_t *program_name; - wchar_t *module_search_path; -} PyPathConfig; - typedef struct { wchar_t *path_env; /* PATH environment variable */ wchar_t *home; /* PYTHONHOME environment variable */ @@ -142,7 +135,7 @@ typedef struct { static const wchar_t delimiter[2] = {DELIM, '\0'}; static const wchar_t separator[2] = {SEP, '\0'}; -static PyPathConfig path_config = {.module_search_path = NULL}; +static _PyPathConfig _Py_path_config = _PyPathConfig_INIT; /* Get file status. Encode the path to the locale encoding. */ @@ -592,10 +585,10 @@ calculate_reduce_exec_prefix(PyCalculatePath *calculate, wchar_t *exec_prefix) static _PyInitError -calculate_program_name(PyCalculatePath *calculate, PyPathConfig *config) +calculate_program_full_path(PyCalculatePath *calculate, _PyPathConfig *config) { - wchar_t program_name[MAXPATHLEN+1]; - memset(program_name, 0, sizeof(program_name)); + wchar_t program_full_path[MAXPATHLEN+1]; + memset(program_full_path, 0, sizeof(program_full_path)); #ifdef __APPLE__ #if MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_4 @@ -612,7 +605,7 @@ calculate_program_name(PyCalculatePath *calculate, PyPathConfig *config) * $PATH isn't exported, you lose. */ if (wcschr(calculate->program_name, SEP)) { - wcsncpy(program_name, calculate->program_name, MAXPATHLEN); + wcsncpy(program_full_path, calculate->program_name, MAXPATHLEN); } #ifdef __APPLE__ /* On Mac OS X, if a script uses an interpreter of the form @@ -628,10 +621,10 @@ calculate_program_name(PyCalculatePath *calculate, PyPathConfig *config) else if(0 == _NSGetExecutablePath(execpath, &nsexeclength) && execpath[0] == SEP) { - size_t r = mbstowcs(program_name, execpath, MAXPATHLEN+1); + size_t r = mbstowcs(program_full_path, execpath, MAXPATHLEN+1); if (r == (size_t)-1 || r > MAXPATHLEN) { /* Could not convert execpath, or it's too long. */ - program_name[0] = '\0'; + program_full_path[0] = '\0'; } } #endif /* __APPLE__ */ @@ -645,34 +638,34 @@ calculate_program_name(PyCalculatePath *calculate, PyPathConfig *config) if (len > MAXPATHLEN) { len = MAXPATHLEN; } - wcsncpy(program_name, path, len); - program_name[len] = '\0'; + wcsncpy(program_full_path, path, len); + program_full_path[len] = '\0'; } else { - wcsncpy(program_name, path, MAXPATHLEN); + wcsncpy(program_full_path, path, MAXPATHLEN); } - joinpath(program_name, calculate->program_name); - if (isxfile(program_name)) { + joinpath(program_full_path, calculate->program_name); + if (isxfile(program_full_path)) { break; } if (!delim) { - program_name[0] = L'\0'; + program_full_path[0] = L'\0'; break; } path = delim + 1; } } else { - program_name[0] = '\0'; + program_full_path[0] = '\0'; } - if (program_name[0] != SEP && program_name[0] != '\0') { - absolutize(program_name); + if (program_full_path[0] != SEP && program_full_path[0] != '\0') { + absolutize(program_full_path); } - config->program_name = _PyMem_RawWcsdup(program_name); - if (config->program_name == NULL) { + config->program_full_path = _PyMem_RawWcsdup(program_full_path); + if (config->program_full_path == NULL) { return _Py_INIT_NO_MEMORY(); } return _Py_INIT_OK(); @@ -680,9 +673,9 @@ calculate_program_name(PyCalculatePath *calculate, PyPathConfig *config) static _PyInitError -calculate_argv0_path(PyCalculatePath *calculate, const wchar_t *program_name) +calculate_argv0_path(PyCalculatePath *calculate, const wchar_t *program_full_path) { - wcsncpy(calculate->argv0_path, program_name, MAXPATHLEN); + wcsncpy(calculate->argv0_path, program_full_path, MAXPATHLEN); calculate->argv0_path[MAXPATHLEN] = '\0'; #ifdef WITH_NEXT_FRAMEWORK @@ -718,10 +711,10 @@ calculate_argv0_path(PyCalculatePath *calculate, const wchar_t *program_name) if (!ismodule(calculate->argv0_path)) { /* We are in the build directory so use the name of the executable - we know that the absolute path is passed */ - wcsncpy(calculate->argv0_path, program_name, MAXPATHLEN); + wcsncpy(calculate->argv0_path, program_full_path, MAXPATHLEN); } else { - /* Use the location of the library as the program_name */ + /* Use the location of the library as the program_full_path */ wcsncpy(calculate->argv0_path, wbuf, MAXPATHLEN); } PyMem_RawFree(wbuf); @@ -730,7 +723,7 @@ calculate_argv0_path(PyCalculatePath *calculate, const wchar_t *program_name) #if HAVE_READLINK wchar_t tmpbuffer[MAXPATHLEN+1]; - int linklen = _Py_wreadlink(program_name, tmpbuffer, MAXPATHLEN); + int linklen = _Py_wreadlink(program_full_path, tmpbuffer, MAXPATHLEN); while (linklen != -1) { if (tmpbuffer[0] == SEP) { /* tmpbuffer should never be longer than MAXPATHLEN, @@ -738,7 +731,7 @@ calculate_argv0_path(PyCalculatePath *calculate, const wchar_t *program_name) wcsncpy(calculate->argv0_path, tmpbuffer, MAXPATHLEN); } else { - /* Interpret relative to program_name */ + /* Interpret relative to program_full_path */ reduce(calculate->argv0_path); joinpath(calculate->argv0_path, tmpbuffer); } @@ -819,7 +812,7 @@ calculate_zip_path(PyCalculatePath *calculate, const wchar_t *prefix) static _PyInitError calculate_module_search_path(PyCalculatePath *calculate, const wchar_t *prefix, const wchar_t *exec_prefix, - PyPathConfig *config) + _PyPathConfig *config) { /* Calculate size of return buffer */ size_t bufsz = 0; @@ -955,14 +948,14 @@ calculate_free(PyCalculatePath *calculate) static _PyInitError -calculate_path_impl(PyCalculatePath *calculate, PyPathConfig *config) +calculate_path_impl(PyCalculatePath *calculate, _PyPathConfig *config) { - _PyInitError err = calculate_program_name(calculate, config); + _PyInitError err = calculate_program_full_path(calculate, config); if (_Py_INIT_FAILED(err)) { return err; } - err = calculate_argv0_path(calculate, config->program_name); + err = calculate_argv0_path(calculate, config->program_full_path); if (_Py_INIT_FAILED(err)) { return err; } @@ -1011,7 +1004,7 @@ calculate_path_impl(PyCalculatePath *calculate, PyPathConfig *config) static void -pathconfig_clear(PyPathConfig *config) +pathconfig_clear(_PyPathConfig *config) { #define CLEAR(ATTR) \ do { \ @@ -1021,7 +1014,7 @@ pathconfig_clear(PyPathConfig *config) CLEAR(config->prefix); CLEAR(config->exec_prefix); - CLEAR(config->program_name); + CLEAR(config->program_full_path); CLEAR(config->module_search_path); #undef CLEAR } @@ -1032,7 +1025,7 @@ pathconfig_clear(PyPathConfig *config) _PyInitError _PyPathConfig_Init(const _PyMainInterpreterConfig *main_config) { - if (path_config.module_search_path) { + if (_Py_path_config.module_search_path) { /* Already initialized */ return _Py_INIT_OK(); } @@ -1045,7 +1038,7 @@ _PyPathConfig_Init(const _PyMainInterpreterConfig *main_config) goto done; } - PyPathConfig new_path_config; + _PyPathConfig new_path_config; memset(&new_path_config, 0, sizeof(new_path_config)); err = calculate_path_impl(&calculate, &new_path_config); @@ -1054,7 +1047,7 @@ _PyPathConfig_Init(const _PyMainInterpreterConfig *main_config) goto done; } - path_config = new_path_config; + _Py_path_config = new_path_config; err = _Py_INIT_OK(); done: @@ -1064,8 +1057,13 @@ _PyPathConfig_Init(const _PyMainInterpreterConfig *main_config) static void -calculate_path(void) +pathconfig_global_init(void) { + if (_Py_path_config.module_search_path) { + /* Already initialized */ + return; + } + _PyInitError err; _PyMainInterpreterConfig config = _PyMainInterpreterConfig_INIT; @@ -1084,7 +1082,7 @@ calculate_path(void) void _PyPathConfig_Fini(void) { - pathconfig_clear(&path_config); + pathconfig_clear(&_Py_path_config); } @@ -1093,58 +1091,50 @@ void Py_SetPath(const wchar_t *path) { if (path == NULL) { - pathconfig_clear(&path_config); + pathconfig_clear(&_Py_path_config); return; } - PyPathConfig new_config; - new_config.program_name = _PyMem_RawWcsdup(Py_GetProgramName()); + _PyPathConfig new_config; + new_config.program_full_path = _PyMem_RawWcsdup(Py_GetProgramName()); new_config.exec_prefix = _PyMem_RawWcsdup(L""); new_config.prefix = _PyMem_RawWcsdup(L""); new_config.module_search_path = _PyMem_RawWcsdup(path); - pathconfig_clear(&path_config); - path_config = new_config; + pathconfig_clear(&_Py_path_config); + _Py_path_config = new_config; } wchar_t * Py_GetPath(void) { - if (!path_config.module_search_path) { - calculate_path(); - } - return path_config.module_search_path; + pathconfig_global_init(); + return _Py_path_config.module_search_path; } wchar_t * Py_GetPrefix(void) { - if (!path_config.module_search_path) { - calculate_path(); - } - return path_config.prefix; + pathconfig_global_init(); + return _Py_path_config.prefix; } wchar_t * Py_GetExecPrefix(void) { - if (!path_config.module_search_path) { - calculate_path(); - } - return path_config.exec_prefix; + pathconfig_global_init(); + return _Py_path_config.exec_prefix; } wchar_t * Py_GetProgramFullPath(void) { - if (!path_config.module_search_path) { - calculate_path(); - } - return path_config.program_name; + pathconfig_global_init(); + return _Py_path_config.program_full_path; } #ifdef __cplusplus diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 4d85f0c368b..7911028ad83 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -564,6 +564,8 @@ PyMem_Free(void *ptr) wchar_t* _PyMem_RawWcsdup(const wchar_t *str) { + assert(str != NULL); + size_t len = wcslen(str); if (len > (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t) - 1) { return NULL; @@ -582,13 +584,12 @@ _PyMem_RawWcsdup(const wchar_t *str) char * _PyMem_RawStrdup(const char *str) { - size_t size; - char *copy; - - size = strlen(str) + 1; - copy = PyMem_RawMalloc(size); - if (copy == NULL) + assert(str != NULL); + size_t size = strlen(str) + 1; + char *copy = PyMem_RawMalloc(size); + if (copy == NULL) { return NULL; + } memcpy(copy, str, size); return copy; } @@ -596,13 +597,12 @@ _PyMem_RawStrdup(const char *str) char * _PyMem_Strdup(const char *str) { - size_t size; - char *copy; - - size = strlen(str) + 1; - copy = PyMem_Malloc(size); - if (copy == NULL) + assert(str != NULL); + size_t size = strlen(str) + 1; + char *copy = PyMem_Malloc(size); + if (copy == NULL) { return NULL; + } memcpy(copy, str, size); return copy; } diff --git a/PC/getpathp.c b/PC/getpathp.c index fe0226ba5d2..ad04b6b6efc 100644 --- a/PC/getpathp.c +++ b/PC/getpathp.c @@ -80,6 +80,7 @@ #include "Python.h" +#include "internal/pystate.h" #include "osdefs.h" #include <wchar.h> @@ -116,13 +117,6 @@ #define LANDMARK L"lib\\os.py" #endif -typedef struct { - wchar_t *prefix; - wchar_t *program_name; - wchar_t *dll_path; - wchar_t *module_search_path; -} PyPathConfig; - typedef struct { wchar_t *module_search_path_env; /* PYTHONPATH environment variable */ wchar_t *path_env; /* PATH environment variable */ @@ -138,7 +132,7 @@ typedef struct { } PyCalculatePath; -static PyPathConfig path_config = {.module_search_path = NULL}; +static _PyPathConfig _Py_path_config = _PyPathConfig_INIT; /* determine if "ch" is a separator character */ @@ -295,7 +289,7 @@ gotlandmark(wchar_t *prefix, const wchar_t *landmark) /* assumes argv0_path is MAXPATHLEN+1 bytes long, already \0 term'd. - assumption provided by only caller, calculate_path() */ + assumption provided by only caller, calculate_path_impl() */ static int search_for_prefix(wchar_t *prefix, wchar_t *argv0_path, const wchar_t *landmark) { @@ -484,7 +478,7 @@ getpythonregpath(HKEY keyBase, int skipcore) static _PyInitError -get_dll_path(PyCalculatePath *calculate, PyPathConfig *config) +get_dll_path(PyCalculatePath *calculate, _PyPathConfig *config) { wchar_t dll_path[MAXPATHLEN+1]; memset(dll_path, 0, sizeof(dll_path)); @@ -509,12 +503,12 @@ get_dll_path(PyCalculatePath *calculate, PyPathConfig *config) static _PyInitError -get_program_name(PyCalculatePath *calculate, PyPathConfig *config) +get_program_full_path(PyCalculatePath *calculate, _PyPathConfig *config) { - wchar_t program_name[MAXPATHLEN+1]; - memset(program_name, 0, sizeof(program_name)); + wchar_t program_full_path[MAXPATHLEN+1]; + memset(program_full_path, 0, sizeof(program_full_path)); - if (GetModuleFileNameW(NULL, program_name, MAXPATHLEN)) { + if (GetModuleFileNameW(NULL, program_full_path, MAXPATHLEN)) { goto done; } @@ -529,7 +523,7 @@ get_program_name(PyCalculatePath *calculate, PyPathConfig *config) if (wcschr(calculate->program_name, SEP)) #endif { - wcsncpy(program_name, calculate->program_name, MAXPATHLEN); + wcsncpy(program_full_path, calculate->program_name, MAXPATHLEN); } else if (calculate->path_env) { wchar_t *path = calculate->path_env; @@ -540,33 +534,33 @@ get_program_name(PyCalculatePath *calculate, PyPathConfig *config) size_t len = delim - path; /* ensure we can't overwrite buffer */ len = min(MAXPATHLEN,len); - wcsncpy(program_name, path, len); - program_name[len] = '\0'; + wcsncpy(program_full_path, path, len); + program_full_path[len] = '\0'; } else { - wcsncpy(program_name, path, MAXPATHLEN); + wcsncpy(program_full_path, path, MAXPATHLEN); } /* join() is safe for MAXPATHLEN+1 size buffer */ - join(program_name, calculate->program_name); - if (exists(program_name)) { + join(program_full_path, calculate->program_name); + if (exists(program_full_path)) { break; } if (!delim) { - program_name[0] = '\0'; + program_full_path[0] = '\0'; break; } path = delim + 1; } } else { - program_name[0] = '\0'; + program_full_path[0] = '\0'; } done: - config->program_name = _PyMem_RawWcsdup(program_name); - if (config->program_name == NULL) { + config->program_full_path = _PyMem_RawWcsdup(program_full_path); + if (config->program_full_path == NULL) { return _Py_INIT_NO_MEMORY(); } return _Py_INIT_OK(); @@ -626,7 +620,7 @@ find_env_config_value(FILE * env_file, const wchar_t * key, wchar_t * value) static int -read_pth_file(PyPathConfig *config, wchar_t *prefix, const wchar_t *path, +read_pth_file(_PyPathConfig *config, wchar_t *prefix, const wchar_t *path, int *isolated, int *nosite) { FILE *sp_file = _Py_wfopen(path, L"r"); @@ -727,15 +721,15 @@ calculate_init(PyCalculatePath *calculate, static int -get_pth_filename(wchar_t *spbuffer, PyPathConfig *config) +get_pth_filename(wchar_t *spbuffer, _PyPathConfig *config) { if (config->dll_path[0]) { if (!change_ext(spbuffer, config->dll_path, L"._pth") && exists(spbuffer)) { return 1; } } - if (config->program_name[0]) { - if (!change_ext(spbuffer, config->program_name, L"._pth") && exists(spbuffer)) { + if (config->program_full_path[0]) { + if (!change_ext(spbuffer, config->program_full_path, L"._pth") && exists(spbuffer)) { return 1; } } @@ -744,7 +738,7 @@ get_pth_filename(wchar_t *spbuffer, PyPathConfig *config) static int -calculate_pth_file(PyPathConfig *config, wchar_t *prefix) +calculate_pth_file(_PyPathConfig *config, wchar_t *prefix) { wchar_t spbuffer[MAXPATHLEN+1]; @@ -821,7 +815,7 @@ calculate_home_prefix(PyCalculatePath *calculate, wchar_t *prefix) static _PyInitError -calculate_module_search_path(PyCalculatePath *calculate, PyPathConfig *config, wchar_t *prefix) +calculate_module_search_path(PyCalculatePath *calculate, _PyPathConfig *config, wchar_t *prefix) { int skiphome = calculate->home==NULL ? 0 : 1; #ifdef Py_ENABLE_SHARED @@ -1002,7 +996,7 @@ calculate_module_search_path(PyCalculatePath *calculate, PyPathConfig *config, w static _PyInitError -calculate_path_impl(PyCalculatePath *calculate, PyPathConfig *config, +calculate_path_impl(PyCalculatePath *calculate, _PyPathConfig *config, const _PyMainInterpreterConfig *main_config) { _PyInitError err; @@ -1012,13 +1006,13 @@ calculate_path_impl(PyCalculatePath *calculate, PyPathConfig *config, return err; } - err = get_program_name(calculate, config); + err = get_program_full_path(calculate, config); if (_Py_INIT_FAILED(err)) { return err; } - /* program_name guaranteed \0 terminated in MAXPATH+1 bytes. */ - wcscpy_s(calculate->argv0_path, MAXPATHLEN+1, config->program_name); + /* program_full_path guaranteed \0 terminated in MAXPATH+1 bytes. */ + wcscpy_s(calculate->argv0_path, MAXPATHLEN+1, config->program_full_path); reduce(calculate->argv0_path); wchar_t prefix[MAXPATHLEN+1]; @@ -1033,7 +1027,7 @@ calculate_path_impl(PyCalculatePath *calculate, PyPathConfig *config, /* Calculate zip archive path from DLL or exe path */ change_ext(calculate->zip_path, - config->dll_path[0] ? config->dll_path : config->program_name, + config->dll_path[0] ? config->dll_path : config->program_full_path, L".zip"); calculate_home_prefix(calculate, prefix); @@ -1062,7 +1056,7 @@ calculate_free(PyCalculatePath *calculate) static void -pathconfig_clear(PyPathConfig *config) +pathconfig_clear(_PyPathConfig *config) { #define CLEAR(ATTR) \ do { \ @@ -1071,7 +1065,7 @@ pathconfig_clear(PyPathConfig *config) } while (0) CLEAR(config->prefix); - CLEAR(config->program_name); + CLEAR(config->program_full_path); CLEAR(config->dll_path); CLEAR(config->module_search_path); #undef CLEAR @@ -1083,7 +1077,7 @@ pathconfig_clear(PyPathConfig *config) _PyInitError _PyPathConfig_Init(const _PyMainInterpreterConfig *main_config) { - if (path_config.module_search_path) { + if (_Py_path_config.module_search_path) { /* Already initialized */ return _Py_INIT_OK(); } @@ -1095,7 +1089,7 @@ _PyPathConfig_Init(const _PyMainInterpreterConfig *main_config) calculate_init(&calculate, main_config); - PyPathConfig new_path_config; + _PyPathConfig new_path_config; memset(&new_path_config, 0, sizeof(new_path_config)); err = calculate_path_impl(&calculate, &new_path_config, main_config); @@ -1103,7 +1097,7 @@ _PyPathConfig_Init(const _PyMainInterpreterConfig *main_config) goto done; } - path_config = new_path_config; + _Py_path_config = new_path_config; err = _Py_INIT_OK(); done: @@ -1116,8 +1110,13 @@ _PyPathConfig_Init(const _PyMainInterpreterConfig *main_config) static void -calculate_path(void) +pathconfig_global_init(void) { + if (_Py_path_config.module_search_path) { + /* Already initialized */ + return; + } + _PyInitError err; _PyMainInterpreterConfig config = _PyMainInterpreterConfig_INIT; @@ -1136,7 +1135,7 @@ calculate_path(void) void _PyPathConfig_Fini(void) { - pathconfig_clear(&path_config); + pathconfig_clear(&_Py_path_config); } @@ -1145,42 +1144,38 @@ _PyPathConfig_Fini(void) void Py_SetPath(const wchar_t *path) { - if (path_config.module_search_path != NULL) { - pathconfig_clear(&path_config); + if (_Py_path_config.module_search_path != NULL) { + pathconfig_clear(&_Py_path_config); } if (path == NULL) { return; } - PyPathConfig new_config; - new_config.program_name = _PyMem_RawWcsdup(Py_GetProgramName()); + _PyPathConfig new_config; + new_config.program_full_path = _PyMem_RawWcsdup(Py_GetProgramName()); new_config.prefix = _PyMem_RawWcsdup(L""); new_config.dll_path = _PyMem_RawWcsdup(L""); new_config.module_search_path = _PyMem_RawWcsdup(path); - pathconfig_clear(&path_config); - path_config = new_config; + pathconfig_clear(&_Py_path_config); + _Py_path_config = new_config; } wchar_t * Py_GetPath(void) { - if (!path_config.module_search_path) { - calculate_path(); - } - return path_config.module_search_path; + pathconfig_global_init(); + return _Py_path_config.module_search_path; } wchar_t * Py_GetPrefix(void) { - if (!path_config.module_search_path) { - calculate_path(); - } - return path_config.prefix; + pathconfig_global_init(); + return _Py_path_config.prefix; } @@ -1194,10 +1189,8 @@ Py_GetExecPrefix(void) wchar_t * Py_GetProgramFullPath(void) { - if (!path_config.module_search_path) { - calculate_path(); - } - return path_config.program_name; + pathconfig_global_init(); + return _Py_path_config.program_full_path; } @@ -1221,7 +1214,7 @@ _Py_CheckPython3() /* If there is a python3.dll next to the python3y.dll, assume this is a build tree; use that DLL */ - wcscpy(py3path, path_config.dll_path); + wcscpy(py3path, _Py_path_config.dll_path); s = wcsrchr(py3path, L'\\'); if (!s) { s = py3path;

1 0

Fix asyncio.streams.FlowControlMixin docstring typo. (#4578)
by Andrew Svetlov Dec. 1, 2017

Dec. 1, 2017

https://github.com/python/cpython/commit/8f5c28b19374df47ebe3d665e54c1dda5c… commit: 8f5c28b19374df47ebe3d665e54c1dda5c2d7dca branch: master author: John Chen <johnchen902(a)gmail.com> committer: Andrew Svetlov <andrew.svetlov(a)gmail.com> date: 2017-12-01T14:33:40+02:00 summary: Fix asyncio.streams.FlowControlMixin docstring typo. (#4578) files: M Lib/asyncio/streams.py diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index 05774e95300..15c9513527f 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -154,7 +154,7 @@ class FlowControlMixin(protocols.Protocol): """Reusable flow control logic for StreamWriter.drain(). This implements the protocol methods pause_writing(), - resume_reading() and connection_lost(). If the subclass overrides + resume_writing() and connection_lost(). If the subclass overrides these it must call the super methods. StreamWriter.drain() must wait for _drain_helper() coroutine.

1 0

Daily reference leaks (4243df51fe43): sum=6
by solipsis＠pitrou.net Dec. 1, 2017

Dec. 1, 2017

results for 4243df51fe43 on branch "default" -------------------------------------------- test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_forkserver leaked [2, 0, 0] memory blocks, sum=2 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/refloghJs4kf', '--timeout', '7200']

1 0

Don't hide unexpected errors in PyErr_WarnExplicitObject(). (GH-4585) (#4662)
by Serhiy Storchaka Dec. 1, 2017

Dec. 1, 2017

https://github.com/python/cpython/commit/9881e4e5386bf6a74b4a542321c23a7c39… commit: 9881e4e5386bf6a74b4a542321c23a7c396035ef branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: Serhiy Storchaka <storchaka(a)gmail.com> date: 2017-12-01T09:21:45+02:00 summary: Don't hide unexpected errors in PyErr_WarnExplicitObject(). (GH-4585) (#4662) (cherry picked from commit a561862048555d555fa4850eaf832ae5474c7e1f) files: M Python/ast.c diff --git a/Python/ast.c b/Python/ast.c index d2710259acb..ede7f4fd990 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -4136,18 +4136,19 @@ warn_invalid_escape_sequence(struct compiling *c, const node *n, } if (PyErr_WarnExplicitObject(PyExc_DeprecationWarning, msg, c->c_filename, LINENO(n), - NULL, NULL) < 0 && - PyErr_ExceptionMatches(PyExc_DeprecationWarning)) + NULL, NULL) < 0) { - const char *s; + if (PyErr_ExceptionMatches(PyExc_DeprecationWarning)) { + const char *s; - /* Replace the DeprecationWarning exception with a SyntaxError - to get a more accurate error report */ - PyErr_Clear(); + /* Replace the DeprecationWarning exception with a SyntaxError + to get a more accurate error report */ + PyErr_Clear(); - s = PyUnicode_AsUTF8(msg); - if (s != NULL) { - ast_error(c, n, s); + s = PyUnicode_AsUTF8(msg); + if (s != NULL) { + ast_error(c, n, s); + } } Py_DECREF(msg); return -1;

1 0

bpo-28668: test.support.requires_multiprocessing_queue is removed (GH-4560)
by xdegaye Dec. 1, 2017

Dec. 1, 2017

https://github.com/python/cpython/commit/bf2b65e413ca5ec705c7e2f463d0d75e94… commit: bf2b65e413ca5ec705c7e2f463d0d75e947588a4 branch: master author: xdegaye <xdegaye(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-12-01T08:08:49+01:00 summary: bpo-28668: test.support.requires_multiprocessing_queue is removed (GH-4560) Skip tests with test.support.import_module('multiprocessing.synchronize') instead when the semaphore implementation is broken or missing. files: A Misc/NEWS.d/next/Tests/2017-11-25-14-53-29.bpo-28668.Y1G6pA.rst M Lib/test/support/__init__.py M Lib/test/test_logging.py diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index f0e15078d62..22868d4ba1a 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -87,8 +87,7 @@ "bigmemtest", "bigaddrspacetest", "cpython_only", "get_attribute", "requires_IEEE_754", "skip_unless_xattr", "requires_zlib", "anticipate_failure", "load_package_tests", "detect_api_mismatch", - "check__all__", "requires_multiprocessing_queue", - "skip_unless_bind_unix_socket", + "check__all__", "skip_unless_bind_unix_socket", # sys "is_jython", "is_android", "check_impl_detail", "unix_shell", "setswitchinterval", @@ -1791,22 +1790,6 @@ def impl_detail(msg=None, **guards): msg = msg.format(' or '.join(guardnames)) return unittest.skip(msg) -_have_mp_queue = None -def requires_multiprocessing_queue(test): - """Skip decorator for tests that use multiprocessing.Queue.""" - global _have_mp_queue - if _have_mp_queue is None: - import multiprocessing - # Without a functioning shared semaphore implementation attempts to - # instantiate a Queue will result in an ImportError (issue #3770). - try: - multiprocessing.Queue() - _have_mp_queue = True - except ImportError: - _have_mp_queue = False - msg = "requires a functioning shared semaphore implementation" - return test if _have_mp_queue else unittest.skip(msg)(test) - def _parse_guards(guards): # Returns a tuple ({platform_name: run_me}, default_value) if not guards: diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 1968b4253c2..7101e3b1576 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -3211,9 +3211,11 @@ def test_handle_called_with_queue_queue(self, mock_handle): self.assertEqual(mock_handle.call_count, 5 * self.repeat, 'correct number of handled log messages') - @support.requires_multiprocessing_queue @patch.object(logging.handlers.QueueListener, 'handle') def test_handle_called_with_mp_queue(self, mock_handle): + # Issue 28668: The multiprocessing (mp) module is not functional + # when the mp.synchronize module cannot be imported. + support.import_module('multiprocessing.synchronize') for i in range(self.repeat): log_queue = multiprocessing.Queue() self.setup_and_log(log_queue, '%s_%s' % (self.id(), i)) @@ -3230,7 +3232,6 @@ def get_all_from_queue(log_queue): except queue.Empty: return [] - @support.requires_multiprocessing_queue def test_no_messages_in_queue_after_stop(self): """ Five messages are logged then the QueueListener is stopped. This @@ -3238,6 +3239,9 @@ def test_no_messages_in_queue_after_stop(self): indicates that messages were not registered on the queue until _after_ the QueueListener stopped. """ + # Issue 28668: The multiprocessing (mp) module is not functional + # when the mp.synchronize module cannot be imported. + support.import_module('multiprocessing.synchronize') for i in range(self.repeat): queue = multiprocessing.Queue() self.setup_and_log(queue, '%s_%s' %(self.id(), i)) diff --git a/Misc/NEWS.d/next/Tests/2017-11-25-14-53-29.bpo-28668.Y1G6pA.rst b/Misc/NEWS.d/next/Tests/2017-11-25-14-53-29.bpo-28668.Y1G6pA.rst new file mode 100644 index 00000000000..e80516d984e --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2017-11-25-14-53-29.bpo-28668.Y1G6pA.rst @@ -0,0 +1,3 @@ +test.support.requires_multiprocessing_queue is removed. Skip tests with +test.support.import_module('multiprocessing.synchronize') instead when the +semaphore implementation is broken or missing.

1 0