Mailman 3 August 2018 - Python-checkins

bpo-34485: Add _Py_ClearStandardStreamEncoding() (GH-8982)
by Victor Stinner Aug. 28, 2018

Aug. 28, 2018

https://github.com/python/cpython/commit/124b9eb4e41ba1676dad954eec9a1fb348… commit: 124b9eb4e41ba1676dad954eec9a1fb3480794e4 branch: master author: Victor Stinner <vstinner(a)redhat.com> committer: GitHub <noreply(a)github.com> date: 2018-08-29T01:29:06+02:00 summary: bpo-34485: Add _Py_ClearStandardStreamEncoding() (GH-8982) * Move Py_SetStandardStreamEncoding() from pylifecycle.c to coreconfig.c * Add _Py_ClearStandardStreamEncoding() private function. * pymain_free() now calls _Py_ClearStandardStreamEncoding(). * Add assertions add the end of _PyCoreConfig_Read() * _PyCoreConfig_Copy(): rename COPY_STR_ATTR() macro to COPY_WSTR_ATTR(). * Fix get_stdio_errors() indentation. files: M Include/pylifecycle.h M Modules/main.c M Python/coreconfig.c M Python/pylifecycle.c diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index 4965d7cbf06c..20298277023f 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -19,7 +19,13 @@ PyAPI_FUNC(wchar_t *) Py_GetPythonHome(void); */ PyAPI_FUNC(int) Py_SetStandardStreamEncoding(const char *encoding, const char *errors); +#endif +#ifdef Py_BUILD_CORE +PyAPI_FUNC(void) _Py_ClearStandardStreamEncoding(void); +#endif + +#ifndef Py_LIMITED_API /* PEP 432 Multi-phase initialization API (Private while provisional!) */ PyAPI_FUNC(_PyInitError) _Py_InitializeCore( PyInterpreterState **interp, diff --git a/Modules/main.c b/Modules/main.c index 82e00936b209..f93ca4d41662 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -466,6 +466,7 @@ pymain_free(_PyMain *pymain) remain valid after Py_Finalize(), since Py_Initialize()-Py_Finalize() can be called multiple times. */ _PyPathConfig_ClearGlobal(); + _Py_ClearStandardStreamEncoding(); /* Force the allocator used by pymain_read_conf() */ PyMemAllocatorEx old_alloc; @@ -1262,7 +1263,6 @@ pymain_read_conf_impl(_PyMain *pymain, _PyCoreConfig *config, return -1; } - assert(config->use_environment >= 0); if (config->use_environment) { err = cmdline_init_env_warnoptions(pymain, config, cmdline); if (_Py_INIT_FAILED(err)) { diff --git a/Python/coreconfig.c b/Python/coreconfig.c index 8724681a79bc..99d703cab92d 100644 --- a/Python/coreconfig.c +++ b/Python/coreconfig.c @@ -85,6 +85,88 @@ _Py_wstrlist_copy(int len, wchar_t **list) } +/* Helper to allow an embedding application to override the normal + * mechanism that attempts to figure out an appropriate IO encoding + */ + +char *_Py_StandardStreamEncoding = NULL; +char *_Py_StandardStreamErrors = NULL; + +int +Py_SetStandardStreamEncoding(const char *encoding, const char *errors) +{ + if (Py_IsInitialized()) { + /* This is too late to have any effect */ + return -1; + } + + int res = 0; + + /* Py_SetStandardStreamEncoding() can be called before Py_Initialize(), + but Py_Initialize() can change the allocator. Use a known allocator + to be able to release the memory later. */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + /* Can't call PyErr_NoMemory() on errors, as Python hasn't been + * initialised yet. + * + * However, the raw memory allocators are initialised appropriately + * as C static variables, so _PyMem_RawStrdup is OK even though + * Py_Initialize hasn't been called yet. + */ + if (encoding) { + _Py_StandardStreamEncoding = _PyMem_RawStrdup(encoding); + if (!_Py_StandardStreamEncoding) { + res = -2; + goto done; + } + } + if (errors) { + _Py_StandardStreamErrors = _PyMem_RawStrdup(errors); + if (!_Py_StandardStreamErrors) { + if (_Py_StandardStreamEncoding) { + PyMem_RawFree(_Py_StandardStreamEncoding); + } + res = -3; + goto done; + } + } +#ifdef MS_WINDOWS + if (_Py_StandardStreamEncoding) { + /* Overriding the stream encoding implies legacy streams */ + Py_LegacyWindowsStdioFlag = 1; + } +#endif + +done: + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + return res; +} + + +void +_Py_ClearStandardStreamEncoding(void) +{ + /* Use the same allocator than Py_SetStandardStreamEncoding() */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + /* We won't need them anymore. */ + if (_Py_StandardStreamEncoding) { + PyMem_RawFree(_Py_StandardStreamEncoding); + _Py_StandardStreamEncoding = NULL; + } + if (_Py_StandardStreamErrors) { + PyMem_RawFree(_Py_StandardStreamErrors); + _Py_StandardStreamErrors = NULL; + } + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); +} + + /* Free memory allocated in config, but don't clear all attributes */ void _PyCoreConfig_Clear(_PyCoreConfig *config) @@ -134,7 +216,7 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2) _PyCoreConfig_Clear(config); #define COPY_ATTR(ATTR) config->ATTR = config2->ATTR -#define COPY_STR_ATTR(ATTR) \ +#define COPY_WSTR_ATTR(ATTR) \ do { \ if (config2->ATTR != NULL) { \ config->ATTR = _PyMem_RawWcsdup(config2->ATTR); \ @@ -173,25 +255,25 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2) COPY_ATTR(coerce_c_locale_warn); COPY_ATTR(utf8_mode); - COPY_STR_ATTR(pycache_prefix); - COPY_STR_ATTR(module_search_path_env); - COPY_STR_ATTR(home); - COPY_STR_ATTR(program_name); - COPY_STR_ATTR(program); + COPY_WSTR_ATTR(pycache_prefix); + COPY_WSTR_ATTR(module_search_path_env); + COPY_WSTR_ATTR(home); + COPY_WSTR_ATTR(program_name); + COPY_WSTR_ATTR(program); COPY_WSTRLIST(argc, argv); COPY_WSTRLIST(nwarnoption, warnoptions); COPY_WSTRLIST(nxoption, xoptions); COPY_WSTRLIST(nmodule_search_path, module_search_paths); - COPY_STR_ATTR(executable); - COPY_STR_ATTR(prefix); - COPY_STR_ATTR(base_prefix); - COPY_STR_ATTR(exec_prefix); + COPY_WSTR_ATTR(executable); + COPY_WSTR_ATTR(prefix); + COPY_WSTR_ATTR(base_prefix); + COPY_WSTR_ATTR(exec_prefix); #ifdef MS_WINDOWS - COPY_STR_ATTR(dll_path); + COPY_WSTR_ATTR(dll_path); #endif - COPY_STR_ATTR(base_exec_prefix); + COPY_WSTR_ATTR(base_exec_prefix); COPY_ATTR(isolated); COPY_ATTR(site_import); @@ -213,7 +295,7 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2) COPY_ATTR(_frozen); #undef COPY_ATTR -#undef COPY_STR_ATTR +#undef COPY_WSTR_ATTR #undef COPY_WSTRLIST return 0; } @@ -627,8 +709,6 @@ get_env_flag(_PyCoreConfig *config, int *flag, const char *name) static _PyInitError config_read_env_vars(_PyCoreConfig *config) { - assert(config->use_environment > 0); - /* Get environment variables */ get_env_flag(config, &config->parser_debug, "PYTHONDEBUG"); get_env_flag(config, &config->verbose, "PYTHONVERBOSE"); @@ -870,6 +950,7 @@ _PyCoreConfig_Read(_PyCoreConfig *config) _PyInitError err; _PyCoreConfig_GetGlobalConfig(config); + assert(config->use_environment >= 0); if (config->isolated > 0) { config->use_environment = 0; @@ -882,7 +963,6 @@ _PyCoreConfig_Read(_PyCoreConfig *config) } #endif - assert(config->use_environment >= 0); if (config->use_environment) { err = config_read_env_vars(config); if (_Py_INIT_FAILED(err)) { @@ -960,12 +1040,12 @@ _PyCoreConfig_Read(_PyCoreConfig *config) if (config->utf8_mode < 0) { config->utf8_mode = 0; } - if (config->_frozen < 0) { - config->_frozen = 0; - } if (config->argc < 0) { config->argc = 0; } + assert(config->coerce_c_locale >= 0); + assert(config->use_environment >= 0); + return _Py_INIT_OK(); } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 4d248ede9ce9..8c77859209ab 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -138,66 +138,6 @@ Py_IsInitialized(void) return _PyRuntime.initialized; } -/* Helper to allow an embedding application to override the normal - * mechanism that attempts to figure out an appropriate IO encoding - */ - -static char *_Py_StandardStreamEncoding = NULL; -static char *_Py_StandardStreamErrors = NULL; - -int -Py_SetStandardStreamEncoding(const char *encoding, const char *errors) -{ - if (Py_IsInitialized()) { - /* This is too late to have any effect */ - return -1; - } - - int res = 0; - - /* Py_SetStandardStreamEncoding() can be called before Py_Initialize(), - but Py_Initialize() can change the allocator. Use a known allocator - to be able to release the memory later. */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - /* Can't call PyErr_NoMemory() on errors, as Python hasn't been - * initialised yet. - * - * However, the raw memory allocators are initialised appropriately - * as C static variables, so _PyMem_RawStrdup is OK even though - * Py_Initialize hasn't been called yet. - */ - if (encoding) { - _Py_StandardStreamEncoding = _PyMem_RawStrdup(encoding); - if (!_Py_StandardStreamEncoding) { - res = -2; - goto done; - } - } - if (errors) { - _Py_StandardStreamErrors = _PyMem_RawStrdup(errors); - if (!_Py_StandardStreamErrors) { - if (_Py_StandardStreamEncoding) { - PyMem_RawFree(_Py_StandardStreamEncoding); - } - res = -3; - goto done; - } - } -#ifdef MS_WINDOWS - if (_Py_StandardStreamEncoding) { - /* Overriding the stream encoding implies legacy streams */ - Py_LegacyWindowsStdioFlag = 1; - } -#endif - -done: - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - return res; -} - /* Global initializations. Can be undone by Py_FinalizeEx(). Don't call this twice without an intervening Py_FinalizeEx() call. When @@ -419,9 +359,9 @@ get_stdio_errors(void) } } #endif - } + } - return "strict"; + return "strict"; } #ifdef PY_COERCE_C_LOCALE @@ -1803,6 +1743,8 @@ init_sys_streams(PyInterpreterState *interp) char *locale_encoding = NULL; char *codec_name = NULL; _PyInitError res = _Py_INIT_OK(); + extern char *_Py_StandardStreamEncoding; + extern char *_Py_StandardStreamErrors; /* Hack to avoid a nasty recursion issue when Python is invoked in verbose mode: pre-import the Latin-1 and UTF-8 codecs */ @@ -1951,22 +1893,8 @@ init_sys_streams(PyInterpreterState *interp) error: res = _Py_INIT_ERR("can't initialize sys standard streams"); - /* Use the same allocator than Py_SetStandardStreamEncoding() */ - PyMemAllocatorEx old_alloc; done: - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - /* We won't need them anymore. */ - if (_Py_StandardStreamEncoding) { - PyMem_RawFree(_Py_StandardStreamEncoding); - _Py_StandardStreamEncoding = NULL; - } - if (_Py_StandardStreamErrors) { - PyMem_RawFree(_Py_StandardStreamErrors); - _Py_StandardStreamErrors = NULL; - } - - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + _Py_ClearStandardStreamEncoding(); PyMem_RawFree(locale_encoding); PyMem_RawFree(codec_name);

1 0

bpo-34485: Fix _Py_InitializeCore() for C locale coercion (GH-8979) (GH-8981)
by Victor Stinner Aug. 28, 2018

Aug. 28, 2018

https://github.com/python/cpython/commit/98c49c6ab239875e35a3c271bc8fabde6c… commit: 98c49c6ab239875e35a3c271bc8fabde6c9be804 branch: 3.7 author: Victor Stinner <vstinner(a)redhat.com> committer: GitHub <noreply(a)github.com> date: 2018-08-29T01:13:29+02:00 summary: bpo-34485: Fix _Py_InitializeCore() for C locale coercion (GH-8979) (GH-8981) * _Py_InitializeCore() now sets the LC_CTYPE locale to the user preferred locale before checking if the C locale should be coerced or not in _PyCoreConfig_Read(). * Fix pymain_read_conf(): remember if the C locale has been coerced when the configuration should be read again if the encoding has changed. (cherry picked from commit 2c8ddcf4f14f3e4c87a6fe6678ab5ad09130c6ab) files: M Modules/main.c M Python/pylifecycle.c diff --git a/Modules/main.c b/Modules/main.c index 856f92c9d9bb..0f7498d6104f 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -1854,16 +1854,18 @@ config_read_env_vars(_PyCoreConfig *config) config->malloc_stats = 1; } - if (config->coerce_c_locale < 0) { - const char *env = config_get_env_var("PYTHONCOERCECLOCALE"); - if (env) { - if (strcmp(env, "0") == 0) { + const char *env = config_get_env_var("PYTHONCOERCECLOCALE"); + if (env) { + if (strcmp(env, "0") == 0) { + if (config->coerce_c_locale < 0) { config->coerce_c_locale = 0; } - else if (strcmp(env, "warn") == 0) { - config->coerce_c_locale_warn = 1; - } - else { + } + else if (strcmp(env, "warn") == 0) { + config->coerce_c_locale_warn = 1; + } + else { + if (config->coerce_c_locale < 0) { config->coerce_c_locale = 1; } } @@ -2044,7 +2046,7 @@ pymain_read_conf(_PyMain *pymain, _Py_CommandLineDetails *cmdline) * See the documentation of the PYTHONCOERCECLOCALE setting for more * details. */ - if (config->coerce_c_locale == 1 && !locale_coerced) { + if (config->coerce_c_locale && !locale_coerced) { locale_coerced = 1; _Py_CoerceLegacyLocale(config); encoding_changed = 1; @@ -2071,6 +2073,7 @@ pymain_read_conf(_PyMain *pymain, _Py_CommandLineDetails *cmdline) pymain_read_conf_impl(). Reset Py_IsolatedFlag and Py_NoSiteFlag modified by _PyCoreConfig_Read(). */ int new_utf8_mode = config->utf8_mode; + int new_coerce_c_locale = config->coerce_c_locale; Py_IgnoreEnvironmentFlag = init_ignore_env; if (_PyCoreConfig_Copy(config, &save_config) < 0) { pymain->err = _Py_INIT_NO_MEMORY(); @@ -2082,6 +2085,7 @@ pymain_read_conf(_PyMain *pymain, _Py_CommandLineDetails *cmdline) cmdline_get_global_config(cmdline); _PyCoreConfig_GetGlobalConfig(config); config->utf8_mode = new_utf8_mode; + config->coerce_c_locale = new_coerce_c_locale; /* The encoding changed: read again the configuration with the new encoding */ diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index f75b8604a330..fc4ee06f144f 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -672,10 +672,6 @@ _Py_InitializeCore_impl(PyInterpreterState **interp_p, _PyRuntime.finalizing = NULL; #ifndef MS_WINDOWS - /* Set up the LC_CTYPE locale, so we can obtain - the locale's charset without having to switch - locales. */ - _Py_SetLocaleFromEnv(LC_CTYPE); _emit_stderr_warning_for_legacy_locale(core_config); #endif @@ -829,6 +825,12 @@ _Py_InitializeCore(PyInterpreterState **interp_p, (and the input configuration is read only). */ _PyCoreConfig config = _PyCoreConfig_INIT; +#ifndef MS_WINDOWS + /* Set up the LC_CTYPE locale, so we can obtain the locale's charset + without having to switch locales. */ + _Py_SetLocaleFromEnv(LC_CTYPE); +#endif + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); if (_PyCoreConfig_Copy(&config, src_config) >= 0) { err = _PyCoreConfig_Read(&config);

1 0

bpo-34485: Fix _Py_InitializeCore() for C locale coercion (GH-8979)
by Victor Stinner Aug. 28, 2018

Aug. 28, 2018

https://github.com/python/cpython/commit/2c8ddcf4f14f3e4c87a6fe6678ab5ad091… commit: 2c8ddcf4f14f3e4c87a6fe6678ab5ad09130c6ab branch: master author: Victor Stinner <vstinner(a)redhat.com> committer: GitHub <noreply(a)github.com> date: 2018-08-29T00:16:53+02:00 summary: bpo-34485: Fix _Py_InitializeCore() for C locale coercion (GH-8979) * _Py_InitializeCore() now sets the LC_CTYPE locale to the user preferred locale before checking if the C locale should be coerced or not in _PyCoreConfig_Read(). * Fix pymain_read_conf(): remember if the C locale has been coerced when the configuration should be read again if the encoding has changed. files: M Modules/main.c M Python/coreconfig.c M Python/pylifecycle.c diff --git a/Modules/main.c b/Modules/main.c index 992d72085f0a..82e00936b209 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -1344,7 +1344,7 @@ pymain_read_conf(_PyMain *pymain, _PyCoreConfig *config, * See the documentation of the PYTHONCOERCECLOCALE setting for more * details. */ - if (config->coerce_c_locale == 1 && !locale_coerced) { + if (config->coerce_c_locale && !locale_coerced) { locale_coerced = 1; _Py_CoerceLegacyLocale(config); encoding_changed = 1; @@ -1369,6 +1369,7 @@ pymain_read_conf(_PyMain *pymain, _PyCoreConfig *config, /* Reset the configuration before reading again the configuration, just keep UTF-8 Mode value. */ int new_utf8_mode = config->utf8_mode; + int new_coerce_c_locale = config->coerce_c_locale; if (_PyCoreConfig_Copy(config, &save_config) < 0) { pymain->err = _Py_INIT_NO_MEMORY(); goto done; @@ -1376,6 +1377,7 @@ pymain_read_conf(_PyMain *pymain, _PyCoreConfig *config, pymain_clear_cmdline(pymain, cmdline); memset(cmdline, 0, sizeof(*cmdline)); config->utf8_mode = new_utf8_mode; + config->coerce_c_locale = new_coerce_c_locale; /* The encoding changed: read again the configuration with the new encoding */ diff --git a/Python/coreconfig.c b/Python/coreconfig.c index d08d75b196ae..8724681a79bc 100644 --- a/Python/coreconfig.c +++ b/Python/coreconfig.c @@ -671,16 +671,18 @@ config_read_env_vars(_PyCoreConfig *config) config->malloc_stats = 1; } - if (config->coerce_c_locale < 0) { - const char *env = _PyCoreConfig_GetEnv(config, "PYTHONCOERCECLOCALE"); - if (env) { - if (strcmp(env, "0") == 0) { + const char *env = _PyCoreConfig_GetEnv(config, "PYTHONCOERCECLOCALE"); + if (env) { + if (strcmp(env, "0") == 0) { + if (config->coerce_c_locale < 0) { config->coerce_c_locale = 0; } - else if (strcmp(env, "warn") == 0) { - config->coerce_c_locale_warn = 1; - } - else { + } + else if (strcmp(env, "warn") == 0) { + config->coerce_c_locale_warn = 1; + } + else { + if (config->coerce_c_locale < 0) { config->coerce_c_locale = 1; } } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 29711dfc982b..4d248ede9ce9 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -659,10 +659,6 @@ _Py_InitializeCore_impl(PyInterpreterState **interp_p, _PyRuntime.finalizing = NULL; #ifndef MS_WINDOWS - /* Set up the LC_CTYPE locale, so we can obtain - the locale's charset without having to switch - locales. */ - _Py_SetLocaleFromEnv(LC_CTYPE); _emit_stderr_warning_for_legacy_locale(core_config); #endif @@ -815,6 +811,12 @@ _Py_InitializeCore(PyInterpreterState **interp_p, (and the input configuration is read only). */ _PyCoreConfig config = _PyCoreConfig_INIT; +#ifndef MS_WINDOWS + /* Set up the LC_CTYPE locale, so we can obtain the locale's charset + without having to switch locales. */ + _Py_SetLocaleFromEnv(LC_CTYPE); +#endif + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); if (_PyCoreConfig_Copy(&config, src_config) >= 0) { err = _PyCoreConfig_Read(&config);

1 0

bpo-34485: Enhance init_sys_streams() (GH-8978)
by Victor Stinner Aug. 28, 2018

Aug. 28, 2018

https://github.com/python/cpython/commit/9e4994d410970fb4e75168401d159ba47a… commit: 9e4994d410970fb4e75168401d159ba47a8f7108 branch: master author: Victor Stinner <vstinner(a)redhat.com> committer: GitHub <noreply(a)github.com> date: 2018-08-28T23:26:33+02:00 summary: bpo-34485: Enhance init_sys_streams() (GH-8978) Python now gets the locale encoding with C code to initialize the encoding of standard streams like sys.stdout. Moreover, the encoding is now initialized to the Python codec name to get a normalized encoding name and to ensure that the codec is loaded. The change avoids importing _bootlocale and _locale modules at startup by default. When the PYTHONIOENCODING environment variable only contains an encoding, the error handler is now is now set explicitly to "strict". Rename also get_default_standard_stream_error_handler() to get_stdio_errors(). Reduce the buffer to format the "cpXXX" string (Windows locale encoding). files: A Misc/NEWS.d/next/Core and Builtins/2018-08-28-17-48-40.bpo-34485.aFwck2.rst A Misc/NEWS.d/next/Core and Builtins/2018-08-28-23-01-14.bpo-34485.dq1Kqk.rst M Lib/test/test_embed.py M Lib/test/test_sys.py M Lib/test/test_utf8_mode.py M Modules/_localemodule.c M Programs/_testembed.c M Python/pylifecycle.c diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 25593bdf4208..3922447c645e 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -171,17 +171,17 @@ def test_forced_io_encoding(self): "stdout: {out_encoding}:ignore", "stderr: {out_encoding}:backslashreplace", "--- Set encoding only ---", - "Expected encoding: latin-1", + "Expected encoding: iso8859-1", "Expected errors: default", - "stdin: latin-1:{errors}", - "stdout: latin-1:{errors}", - "stderr: latin-1:backslashreplace", + "stdin: iso8859-1:{errors}", + "stdout: iso8859-1:{errors}", + "stderr: iso8859-1:backslashreplace", "--- Set encoding and errors ---", - "Expected encoding: latin-1", + "Expected encoding: iso8859-1", "Expected errors: replace", - "stdin: latin-1:replace", - "stdout: latin-1:replace", - "stderr: latin-1:backslashreplace"]) + "stdin: iso8859-1:replace", + "stdout: iso8859-1:replace", + "stderr: iso8859-1:backslashreplace"]) expected_output = expected_output.format( in_encoding=expected_stream_encoding, out_encoding=expected_stream_encoding, diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 336ae447a8de..005c82d13dc7 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -668,7 +668,7 @@ def c_locale_get_error_handler(self, isolated=False, encoding=None): 'dump("stdout")', 'dump("stderr")', )) - args = [sys.executable, "-c", code] + args = [sys.executable, "-X", "utf8=0", "-c", code] if isolated: args.append("-I") if encoding is not None: @@ -712,8 +712,8 @@ def test_c_locale_surrogateescape(self): # have no any effect out = self.c_locale_get_error_handler(encoding=':') self.assertEqual(out, - 'stdin: strict\n' - 'stdout: strict\n' + 'stdin: surrogateescape\n' + 'stdout: surrogateescape\n' 'stderr: backslashreplace\n') out = self.c_locale_get_error_handler(encoding='') self.assertEqual(out, diff --git a/Lib/test/test_utf8_mode.py b/Lib/test/test_utf8_mode.py index df988c1fc9e2..7280ce77ef82 100644 --- a/Lib/test/test_utf8_mode.py +++ b/Lib/test/test_utf8_mode.py @@ -139,16 +139,16 @@ def test_stdio(self): out = self.get_output('-X', 'utf8', '-c', code, PYTHONIOENCODING="latin1") self.assertEqual(out.splitlines(), - ['stdin: latin1/strict', - 'stdout: latin1/strict', - 'stderr: latin1/backslashreplace']) + ['stdin: iso8859-1/strict', + 'stdout: iso8859-1/strict', + 'stderr: iso8859-1/backslashreplace']) out = self.get_output('-X', 'utf8', '-c', code, PYTHONIOENCODING=":namereplace") self.assertEqual(out.splitlines(), - ['stdin: UTF-8/namereplace', - 'stdout: UTF-8/namereplace', - 'stderr: UTF-8/backslashreplace']) + ['stdin: utf-8/namereplace', + 'stdout: utf-8/namereplace', + 'stderr: utf-8/backslashreplace']) def test_io(self): code = textwrap.dedent(''' diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-08-28-17-48-40.bpo-34485.aFwck2.rst b/Misc/NEWS.d/next/Core and Builtins/2018-08-28-17-48-40.bpo-34485.aFwck2.rst new file mode 100644 index 000000000000..f6cd9515f2af --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-08-28-17-48-40.bpo-34485.aFwck2.rst @@ -0,0 +1,5 @@ +Python now gets the locale encoding with C code to initialize the encoding +of standard streams like sys.stdout. Moreover, the encoding is now +initialized to the Python codec name to get a normalized encoding name and +to ensure that the codec is loaded. The change avoids importing _bootlocale +and _locale modules at startup by default. diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-08-28-23-01-14.bpo-34485.dq1Kqk.rst b/Misc/NEWS.d/next/Core and Builtins/2018-08-28-23-01-14.bpo-34485.dq1Kqk.rst new file mode 100644 index 000000000000..5ca373aeab6d --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-08-28-23-01-14.bpo-34485.dq1Kqk.rst @@ -0,0 +1,3 @@ +Fix the error handler of standard streams like sys.stdout: +PYTHONIOENCODING=":" is now ignored instead of setting the error handler to +"strict". diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index 524886d46600..3fdbc5ea8122 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -319,7 +319,7 @@ PyLocale_strxfrm(PyObject* self, PyObject* args) static PyObject* PyLocale_getdefaultlocale(PyObject* self, PyObject *Py_UNUSED(ignored)) { - char encoding[100]; + char encoding[20]; char locale[100]; PyOS_snprintf(encoding, sizeof(encoding), "cp%d", GetACP()); diff --git a/Programs/_testembed.c b/Programs/_testembed.c index f1d30f6c54dd..d0c00cfc6cd4 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -113,9 +113,9 @@ static int test_forced_io_encoding(void) printf("--- Set errors only ---\n"); check_stdio_details(NULL, "ignore"); printf("--- Set encoding only ---\n"); - check_stdio_details("latin-1", NULL); + check_stdio_details("iso8859-1", NULL); printf("--- Set encoding and errors ---\n"); - check_stdio_details("latin-1", "replace"); + check_stdio_details("iso8859-1", "replace"); /* Check calling after initialization fails */ Py_Initialize(); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index cc64cf956d28..29711dfc982b 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -244,22 +244,26 @@ get_codec_name(const char *encoding) return NULL; } -static char* -get_locale_encoding(void) +static _PyInitError +get_locale_encoding(char **locale_encoding) { -#if defined(HAVE_LANGINFO_H) && defined(CODESET) - char* codeset = nl_langinfo(CODESET); - if (!codeset || codeset[0] == '\0') { - PyErr_SetString(PyExc_ValueError, "CODESET is not set or empty"); - return NULL; - } - return get_codec_name(codeset); +#ifdef MS_WINDOWS + char encoding[20]; + PyOS_snprintf(encoding, sizeof(encoding), "cp%d", GetACP()); #elif defined(__ANDROID__) - return get_codec_name("UTF-8"); + const char *encoding = "UTF-8"; #else - PyErr_SetNone(PyExc_NotImplementedError); - return NULL; + const char *encoding = nl_langinfo(CODESET); + if (!encoding || encoding[0] == '\0') { + return _Py_INIT_USER_ERR("failed to get the locale encoding: " + "nl_langinfo(CODESET) failed"); + } #endif + *locale_encoding = _PyMem_RawStrdup(encoding); + if (*locale_encoding == NULL) { + return _Py_INIT_NO_MEMORY(); + } + return _Py_INIT_OK(); } static _PyInitError @@ -397,7 +401,7 @@ static _LocaleCoercionTarget _TARGET_LOCALES[] = { }; static const char * -get_default_standard_stream_error_handler(void) +get_stdio_errors(void) { const char *ctype_loc = setlocale(LC_CTYPE, NULL); if (ctype_loc != NULL) { @@ -417,8 +421,7 @@ get_default_standard_stream_error_handler(void) #endif } - /* Otherwise return NULL to request the typical default error handler */ - return NULL; + return "strict"; } #ifdef PY_COERCE_C_LOCALE @@ -1586,9 +1589,17 @@ initfsencoding(PyInterpreterState *interp) Py_HasFileSystemDefaultEncoding = 1; } else { - Py_FileSystemDefaultEncoding = get_locale_encoding(); + char *locale_encoding; + _PyInitError err = get_locale_encoding(&locale_encoding); + if (_Py_INIT_FAILED(err)) { + return err; + } + + Py_FileSystemDefaultEncoding = get_codec_name(locale_encoding); + PyMem_RawFree(locale_encoding); if (Py_FileSystemDefaultEncoding == NULL) { - return _Py_INIT_ERR("Unable to get the locale encoding"); + return _Py_INIT_ERR("failed to get the Python codec " + "of the locale encoding"); } Py_HasFileSystemDefaultEncoding = 0; @@ -1787,6 +1798,8 @@ init_sys_streams(PyInterpreterState *interp) PyObject * encoding_attr; char *pythonioencoding = NULL; const char *encoding, *errors; + char *locale_encoding = NULL; + char *codec_name = NULL; _PyInitError res = _Py_INIT_OK(); /* Hack to avoid a nasty recursion issue when Python is invoked @@ -1838,21 +1851,46 @@ init_sys_streams(PyInterpreterState *interp) errors = err; } } - if (*pythonioencoding && !encoding) { + if (!encoding && *pythonioencoding) { encoding = pythonioencoding; + if (!errors) { + errors = "strict"; + } } } - else if (interp->core_config.utf8_mode) { - encoding = "utf-8"; - errors = "surrogateescape"; + + if (interp->core_config.utf8_mode) { + if (!encoding) { + encoding = "utf-8"; + } + if (!errors) { + errors = "surrogateescape"; + } } - if (!errors && !pythonioencoding) { + if (!errors) { /* Choose the default error handler based on the current locale */ - errors = get_default_standard_stream_error_handler(); + errors = get_stdio_errors(); } } + if (encoding == NULL) { + _PyInitError err = get_locale_encoding(&locale_encoding); + if (_Py_INIT_FAILED(err)) { + return err; + } + encoding = locale_encoding; + } + + codec_name = get_codec_name(encoding); + if (codec_name == NULL) { + PyErr_SetString(PyExc_RuntimeError, + "failed to get the Python codec name " + "of stdio encoding"); + goto error; + } + encoding = codec_name; + /* Set sys.stdin */ fd = fileno(stdin); /* Under some conditions stdin, stdout and stderr may not be connected @@ -1928,6 +1966,8 @@ init_sys_streams(PyInterpreterState *interp) PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + PyMem_RawFree(locale_encoding); + PyMem_RawFree(codec_name); PyMem_Free(pythonioencoding); Py_XDECREF(bimod); Py_XDECREF(iomod);

1 0

bpo-34403: On HP-UX, force ASCII for C locale (GH-8969)
by Victor Stinner Aug. 28, 2018

Aug. 28, 2018

https://github.com/python/cpython/commit/d500e5307aec9c5d535f66d567fadb9c58… commit: d500e5307aec9c5d535f66d567fadb9c587a9a36 branch: master author: Victor Stinner <vstinner(a)redhat.com> committer: GitHub <noreply(a)github.com> date: 2018-08-28T17:27:36+02:00 summary: bpo-34403: On HP-UX, force ASCII for C locale (GH-8969) On HP-UX with C or POSIX locale, sys.getfilesystemencoding() now returns "ascii" instead of "roman8" (when the UTF-8 Mode is disabled and the C locale is not coerced). nl_langinfo(CODESET) announces "roman8" whereas it uses the Latin1 encoding in practice. files: A Misc/NEWS.d/next/Core and Builtins/2018-08-28-10-49-55.bpo-34403.4Q3LzP.rst M Include/fileutils.h M Python/coreconfig.c M Python/fileutils.c M Python/pylifecycle.c diff --git a/Include/fileutils.h b/Include/fileutils.h index e4bf6d4db95d..370878469df7 100644 --- a/Include/fileutils.h +++ b/Include/fileutils.h @@ -170,6 +170,11 @@ PyAPI_FUNC(int) _Py_GetLocaleconvNumeric( #endif /* Py_LIMITED_API */ + +#ifdef Py_BUILD_CORE +PyAPI_FUNC(int) _Py_GetForceASCII(void); +#endif + #ifdef __cplusplus } #endif diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-08-28-10-49-55.bpo-34403.4Q3LzP.rst b/Misc/NEWS.d/next/Core and Builtins/2018-08-28-10-49-55.bpo-34403.4Q3LzP.rst new file mode 100644 index 000000000000..d70be82b0ab4 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-08-28-10-49-55.bpo-34403.4Q3LzP.rst @@ -0,0 +1,3 @@ +On HP-UX with C or POSIX locale, sys.getfilesystemencoding() now returns +"ascii" instead of "roman8" (when the UTF-8 Mode is disabled and the C locale +is not coerced). diff --git a/Python/coreconfig.c b/Python/coreconfig.c index acf46451f15a..d08d75b196ae 100644 --- a/Python/coreconfig.c +++ b/Python/coreconfig.c @@ -828,18 +828,21 @@ config_read_complex_options(_PyCoreConfig *config) static void config_init_locale(_PyCoreConfig *config) { - if (_Py_LegacyLocaleDetected()) { + if (config->coerce_c_locale < 0) { /* The C locale enables the C locale coercion (PEP 538) */ - if (config->coerce_c_locale < 0) { + if (_Py_LegacyLocaleDetected()) { config->coerce_c_locale = 1; } } + #ifndef MS_WINDOWS - const char *ctype_loc = setlocale(LC_CTYPE, NULL); - if (ctype_loc != NULL - && (strcmp(ctype_loc, "C") == 0 || strcmp(ctype_loc, "POSIX") == 0)) { + if (config->utf8_mode < 0) { /* The C locale and the POSIX locale enable the UTF-8 Mode (PEP 540) */ - if (config->utf8_mode < 0) { + const char *ctype_loc = setlocale(LC_CTYPE, NULL); + if (ctype_loc != NULL + && (strcmp(ctype_loc, "C") == 0 + || strcmp(ctype_loc, "POSIX") == 0)) + { config->utf8_mode = 1; } } diff --git a/Python/fileutils.c b/Python/fileutils.c index b413f4e1e682..e756c260cdcc 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -72,8 +72,8 @@ _Py_device_encoding(int fd) extern int _Py_normalize_encoding(const char *, char *, size_t); -/* Workaround FreeBSD and OpenIndiana locale encoding issue with the C locale. - On these operating systems, nl_langinfo(CODESET) announces an alias of the +/* Workaround FreeBSD and OpenIndiana locale encoding issue with the C locale + and POSIX locale. nl_langinfo(CODESET) announces an alias of the ASCII encoding, whereas mbstowcs() and wcstombs() functions use the ISO-8859-1 encoding. The problem is that os.fsencode() and os.fsdecode() use locale.getpreferredencoding() codec. For example, if command line arguments @@ -86,6 +86,10 @@ extern int _Py_normalize_encoding(const char *, char *, size_t); workaround is also enabled on error, for example if getting the locale failed. + On HP-UX with the C locale or the POSIX locale, nl_langinfo(CODESET) + announces "roman8" but mbstowcs() uses Latin1 in practice. Force also the + ASCII encoding in this case. + Values of force_ascii: 1: the workaround is used: Py_EncodeLocale() uses @@ -100,13 +104,46 @@ static int force_ascii = -1; static int check_force_ascii(void) { - char *loc; + char *loc = setlocale(LC_CTYPE, NULL); + if (loc == NULL) { + goto error; + } + if (strcmp(loc, "C") != 0 && strcmp(loc, "POSIX") != 0) { + /* the LC_CTYPE locale is different than C and POSIX */ + return 0; + } + #if defined(HAVE_LANGINFO_H) && defined(CODESET) - char *codeset, **alias; + const char *codeset = nl_langinfo(CODESET); + if (!codeset || codeset[0] == '\0') { + /* CODESET is not set or empty */ + goto error; + } + char encoding[20]; /* longest name: "iso_646.irv_1991\0" */ - int is_ascii; - unsigned int i; - char* ascii_aliases[] = { + if (!_Py_normalize_encoding(codeset, encoding, sizeof(encoding))) { + goto error; + } + +#ifdef __hpux + if (strcmp(encoding, "roman8") == 0) { + unsigned char ch; + wchar_t wch; + size_t res; + + ch = (unsigned char)0xA7; + res = mbstowcs(&wch, (char*)&ch, 1); + if (res != (size_t)-1 && wch == L'\xA7') { + /* On HP-UX withe C locale or the POSIX locale, + nl_langinfo(CODESET) announces "roman8", whereas mbstowcs() uses + Latin1 encoding in practice. Force ASCII in this case. + + Roman8 decodes 0xA7 to U+00CF. Latin1 decodes 0xA7 to U+00A7. */ + return 1; + } + } +#else + const char* ascii_aliases[] = { "ascii", /* Aliases from Lib/encodings/aliases.py */ "646", @@ -123,27 +160,9 @@ check_force_ascii(void) "us_ascii", NULL }; -#endif - - loc = setlocale(LC_CTYPE, NULL); - if (loc == NULL) - goto error; - if (strcmp(loc, "C") != 0 && strcmp(loc, "POSIX") != 0) { - /* the LC_CTYPE locale is different than C */ - return 0; - } - -#if defined(HAVE_LANGINFO_H) && defined(CODESET) - codeset = nl_langinfo(CODESET); - if (!codeset || codeset[0] == '\0') { - /* CODESET is not set or empty */ - goto error; - } - if (!_Py_normalize_encoding(codeset, encoding, sizeof(encoding))) - goto error; - is_ascii = 0; - for (alias=ascii_aliases; *alias != NULL; alias++) { + int is_ascii = 0; + for (const char **alias=ascii_aliases; *alias != NULL; alias++) { if (strcmp(encoding, *alias) == 0) { is_ascii = 1; break; @@ -154,13 +173,14 @@ check_force_ascii(void) return 0; } - for (i=0x80; i<0xff; i++) { - unsigned char ch; - wchar_t wch; + for (unsigned int i=0x80; i<=0xff; i++) { + char ch[1]; + wchar_t wch[1]; size_t res; - ch = (unsigned char)i; - res = mbstowcs(&wch, (char*)&ch, 1); + unsigned uch = (unsigned char)i; + ch[0] = (char)uch; + res = mbstowcs(wch, ch, 1); if (res != (size_t)-1) { /* decoding a non-ASCII character from the locale encoding succeed: the locale encoding is not ASCII, force ASCII */ @@ -169,17 +189,29 @@ check_force_ascii(void) } /* None of the bytes in the range 0x80-0xff can be decoded from the locale encoding: the locale encoding is really ASCII */ +#endif /* !defined(__hpux) */ return 0; #else /* nl_langinfo(CODESET) is not available: always force ASCII */ return 1; -#endif +#endif /* defined(HAVE_LANGINFO_H) && defined(CODESET) */ error: /* if an error occurred, force the ASCII encoding */ return 1; } + +int +_Py_GetForceASCII(void) +{ + if (force_ascii == -1) { + force_ascii = check_force_ascii(); + } + return force_ascii; +} + + static int encode_ascii(const wchar_t *text, char **str, size_t *error_pos, const char **reason, @@ -234,6 +266,12 @@ encode_ascii(const wchar_t *text, char **str, *str = result; return 0; } +#else +int +_Py_GetForceASCII(void) +{ + return 0; +} #endif /* !defined(__APPLE__) && !defined(__ANDROID__) && !defined(MS_WINDOWS) */ diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 28704c1c228b..cc64cf956d28 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1576,21 +1576,25 @@ initfsencoding(PyInterpreterState *interp) Py_FileSystemDefaultEncodeErrors = "surrogatepass"; } #else - if (Py_FileSystemDefaultEncoding == NULL && - interp->core_config.utf8_mode) - { - Py_FileSystemDefaultEncoding = "utf-8"; - Py_HasFileSystemDefaultEncoding = 1; - } - else if (Py_FileSystemDefaultEncoding == NULL) { - Py_FileSystemDefaultEncoding = get_locale_encoding(); - if (Py_FileSystemDefaultEncoding == NULL) { - return _Py_INIT_ERR("Unable to get the locale encoding"); + if (Py_FileSystemDefaultEncoding == NULL) { + if (interp->core_config.utf8_mode) { + Py_FileSystemDefaultEncoding = "utf-8"; + Py_HasFileSystemDefaultEncoding = 1; + } + else if (_Py_GetForceASCII()) { + Py_FileSystemDefaultEncoding = "ascii"; + Py_HasFileSystemDefaultEncoding = 1; } + else { + Py_FileSystemDefaultEncoding = get_locale_encoding(); + if (Py_FileSystemDefaultEncoding == NULL) { + return _Py_INIT_ERR("Unable to get the locale encoding"); + } - Py_HasFileSystemDefaultEncoding = 0; - interp->fscodec_initialized = 1; - return _Py_INIT_OK(); + Py_HasFileSystemDefaultEncoding = 0; + interp->fscodec_initialized = 1; + return _Py_INIT_OK(); + } } #endif

1 0

bpo-34527: Fix FreeBSD with POSIX locale (GH-8975)
by Victor Stinner Aug. 28, 2018

Aug. 28, 2018

https://github.com/python/cpython/commit/4ff38870b1de8a3add5357edf125c2866b… commit: 4ff38870b1de8a3add5357edf125c2866bc42b54 branch: 3.6 author: Victor Stinner <vstinner(a)redhat.com> committer: GitHub <noreply(a)github.com> date: 2018-08-28T14:53:31+02:00 summary: bpo-34527: Fix FreeBSD with POSIX locale (GH-8975) On FreeBSD, Py_DecodeLocale() and Py_EncodeLocale() now also forces the ASCII encoding if the LC_CTYPE locale is "POSIX", not only if the LC_CTYPE locale is "C". files: A Misc/NEWS.d/next/Core and Builtins/2018-08-28-11-53-39.bpo-34527.aBEX9b.rst M Python/fileutils.c diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-08-28-11-53-39.bpo-34527.aBEX9b.rst b/Misc/NEWS.d/next/Core and Builtins/2018-08-28-11-53-39.bpo-34527.aBEX9b.rst new file mode 100644 index 000000000000..9fce794305cc --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-08-28-11-53-39.bpo-34527.aBEX9b.rst @@ -0,0 +1,3 @@ +On FreeBSD, Py_DecodeLocale() and Py_EncodeLocale() now also forces the +ASCII encoding if the LC_CTYPE locale is "POSIX", not only if the LC_CTYPE +locale is "C". diff --git a/Python/fileutils.c b/Python/fileutils.c index b9638d2df513..b8e489125e3c 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -132,7 +132,7 @@ check_force_ascii(void) loc = setlocale(LC_CTYPE, NULL); if (loc == NULL) goto error; - if (strcmp(loc, "C") != 0) { + if (strcmp(loc, "C") != 0 && strcmp(loc, "POSIX") != 0) { /* the LC_CTYPE locale is different than C */ return 0; }

1 0

bpo-34527: POSIX locale enables the UTF-8 Mode (GH-8972) (GH-8974)
by Victor Stinner Aug. 28, 2018

Aug. 28, 2018

https://github.com/python/cpython/commit/65ef7425a32ee411d8047a4fad0fc6bb9f… commit: 65ef7425a32ee411d8047a4fad0fc6bb9ff733b1 branch: 3.7 author: Victor Stinner <vstinner(a)redhat.com> committer: GitHub <noreply(a)github.com> date: 2018-08-28T13:51:20+02:00 summary: bpo-34527: POSIX locale enables the UTF-8 Mode (GH-8972) (GH-8974) * The UTF-8 Mode is now also enabled by the "POSIX" locale, not only by the "C" locale. * On FreeBSD, Py_DecodeLocale() and Py_EncodeLocale() now also forces the ASCII encoding if the LC_CTYPE locale is "POSIX", not only if the LC_CTYPE locale is "C". * test_utf8_mode.test_cmd_line() checks also that the command line arguments are decoded from UTF-8 when the the UTF-8 Mode is enabled with POSIX locale or C locale. (cherry picked from commit 5cb258950ce9b69b1f65646431c464c0c17b1510) files: A Misc/NEWS.d/next/Core and Builtins/2018-08-28-11-52-13.bpo-34527.sh5MQJ.rst A Misc/NEWS.d/next/Core and Builtins/2018-08-28-11-53-39.bpo-34527.aBEX9b.rst M Lib/test/test_utf8_mode.py M Modules/main.c M Python/fileutils.c diff --git a/Lib/test/test_utf8_mode.py b/Lib/test/test_utf8_mode.py index 26e2e13ec533..4a16b7304689 100644 --- a/Lib/test/test_utf8_mode.py +++ b/Lib/test/test_utf8_mode.py @@ -12,6 +12,7 @@ MS_WINDOWS = (sys.platform == 'win32') +POSIX_LOCALES = ('C', 'POSIX') class UTF8ModeTests(unittest.TestCase): @@ -23,7 +24,7 @@ class UTF8ModeTests(unittest.TestCase): def posix_locale(self): loc = locale.setlocale(locale.LC_CTYPE, None) - return (loc == 'C') + return (loc in POSIX_LOCALES) def get_output(self, *args, failure=False, **kw): kw = dict(self.DEFAULT_ENV, **kw) @@ -39,8 +40,10 @@ def get_output(self, *args, failure=False, **kw): def test_posix_locale(self): code = 'import sys; print(sys.flags.utf8_mode)' - out = self.get_output('-c', code, LC_ALL='C') - self.assertEqual(out, '1') + for loc in POSIX_LOCALES: + with self.subTest(LC_ALL=loc): + out = self.get_output('-c', code, LC_ALL=loc) + self.assertEqual(out, '1') def test_xoption(self): code = 'import sys; print(sys.flags.utf8_mode)' @@ -201,8 +204,10 @@ def test_locale_getpreferredencoding(self): out = self.get_output('-X', 'utf8', '-c', code) self.assertEqual(out, 'UTF-8 UTF-8') - out = self.get_output('-X', 'utf8', '-c', code, LC_ALL='C') - self.assertEqual(out, 'UTF-8 UTF-8') + for loc in POSIX_LOCALES: + with self.subTest(LC_ALL=loc): + out = self.get_output('-X', 'utf8', '-c', code, LC_ALL=loc) + self.assertEqual(out, 'UTF-8 UTF-8') @unittest.skipIf(MS_WINDOWS, 'test specific to Unix') def test_cmd_line(self): @@ -217,11 +222,17 @@ def check(utf8_opt, expected, **kw): self.assertEqual(args, ascii(expected), out) check('utf8', [arg_utf8]) + for loc in POSIX_LOCALES: + with self.subTest(LC_ALL=loc): + check('utf8', [arg_utf8], LC_ALL=loc) + if sys.platform == 'darwin' or support.is_android: c_arg = arg_utf8 else: c_arg = arg_ascii - check('utf8=0', [c_arg], LC_ALL='C') + for loc in POSIX_LOCALES: + with self.subTest(LC_ALL=loc): + check('utf8=0', [c_arg], LC_ALL=loc) def test_optim_level(self): # CPython: check that Py_Main() doesn't increment Py_OptimizeFlag diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-08-28-11-52-13.bpo-34527.sh5MQJ.rst b/Misc/NEWS.d/next/Core and Builtins/2018-08-28-11-52-13.bpo-34527.sh5MQJ.rst new file mode 100644 index 000000000000..280a8922edc3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-08-28-11-52-13.bpo-34527.sh5MQJ.rst @@ -0,0 +1,2 @@ +The UTF-8 Mode is now also enabled by the "POSIX" locale, not only by the "C" +locale. diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-08-28-11-53-39.bpo-34527.aBEX9b.rst b/Misc/NEWS.d/next/Core and Builtins/2018-08-28-11-53-39.bpo-34527.aBEX9b.rst new file mode 100644 index 000000000000..9fce794305cc --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-08-28-11-53-39.bpo-34527.aBEX9b.rst @@ -0,0 +1,3 @@ +On FreeBSD, Py_DecodeLocale() and Py_EncodeLocale() now also forces the +ASCII encoding if the LC_CTYPE locale is "POSIX", not only if the LC_CTYPE +locale is "C". diff --git a/Modules/main.c b/Modules/main.c index 45148dc98391..856f92c9d9bb 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -2102,15 +2102,25 @@ pymain_read_conf(_PyMain *pymain, _Py_CommandLineDetails *cmdline) static void config_init_locale(_PyCoreConfig *config) { - if (_Py_LegacyLocaleDetected()) { - /* POSIX locale: enable C locale coercion and UTF-8 Mode */ - if (config->utf8_mode < 0) { - config->utf8_mode = 1; - } - if (config->coerce_c_locale < 0) { + if (config->coerce_c_locale < 0) { + /* The C locale enables the C locale coercion (PEP 538) */ + if (_Py_LegacyLocaleDetected()) { config->coerce_c_locale = 1; } } + +#ifndef MS_WINDOWS + if (config->utf8_mode < 0) { + /* The C locale and the POSIX locale enable the UTF-8 Mode (PEP 540) */ + const char *ctype_loc = setlocale(LC_CTYPE, NULL); + if (ctype_loc != NULL + && (strcmp(ctype_loc, "C") == 0 + || strcmp(ctype_loc, "POSIX") == 0)) + { + config->utf8_mode = 1; + } + } +#endif } diff --git a/Python/fileutils.c b/Python/fileutils.c index 35869c81ac9f..b413f4e1e682 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -128,7 +128,7 @@ check_force_ascii(void) loc = setlocale(LC_CTYPE, NULL); if (loc == NULL) goto error; - if (strcmp(loc, "C") != 0) { + if (strcmp(loc, "C") != 0 && strcmp(loc, "POSIX") != 0) { /* the LC_CTYPE locale is different than C */ return 0; }

1 0

bpo-34527: POSIX locale enables the UTF-8 Mode (GH-8972)
by Victor Stinner Aug. 28, 2018

Aug. 28, 2018

https://github.com/python/cpython/commit/5cb258950ce9b69b1f65646431c464c0c1… commit: 5cb258950ce9b69b1f65646431c464c0c17b1510 branch: master author: Victor Stinner <vstinner(a)redhat.com> committer: GitHub <noreply(a)github.com> date: 2018-08-28T12:35:44+02:00 summary: bpo-34527: POSIX locale enables the UTF-8 Mode (GH-8972) * The UTF-8 Mode is now also enabled by the "POSIX" locale, not only by the "C" locale. * On FreeBSD, Py_DecodeLocale() and Py_EncodeLocale() now also forces the ASCII encoding if the LC_CTYPE locale is "POSIX", not only if the LC_CTYPE locale is "C". * test_utf8_mode.test_cmd_line() checks also that the command line arguments are decoded from UTF-8 when the the UTF-8 Mode is enabled with POSIX locale or C locale. files: A Misc/NEWS.d/next/Core and Builtins/2018-08-28-11-52-13.bpo-34527.sh5MQJ.rst A Misc/NEWS.d/next/Core and Builtins/2018-08-28-11-53-39.bpo-34527.aBEX9b.rst M Lib/test/test_utf8_mode.py M Python/coreconfig.c M Python/fileutils.c diff --git a/Lib/test/test_utf8_mode.py b/Lib/test/test_utf8_mode.py index 3e918fd54ce3..df988c1fc9e2 100644 --- a/Lib/test/test_utf8_mode.py +++ b/Lib/test/test_utf8_mode.py @@ -12,6 +12,7 @@ MS_WINDOWS = (sys.platform == 'win32') +POSIX_LOCALES = ('C', 'POSIX') class UTF8ModeTests(unittest.TestCase): @@ -23,7 +24,7 @@ class UTF8ModeTests(unittest.TestCase): def posix_locale(self): loc = locale.setlocale(locale.LC_CTYPE, None) - return (loc == 'C') + return (loc in POSIX_LOCALES) def get_output(self, *args, failure=False, **kw): kw = dict(self.DEFAULT_ENV, **kw) @@ -39,8 +40,10 @@ def get_output(self, *args, failure=False, **kw): def test_posix_locale(self): code = 'import sys; print(sys.flags.utf8_mode)' - out = self.get_output('-c', code, LC_ALL='C') - self.assertEqual(out, '1') + for loc in POSIX_LOCALES: + with self.subTest(LC_ALL=loc): + out = self.get_output('-c', code, LC_ALL=loc) + self.assertEqual(out, '1') def test_xoption(self): code = 'import sys; print(sys.flags.utf8_mode)' @@ -201,8 +204,10 @@ def test_locale_getpreferredencoding(self): out = self.get_output('-X', 'utf8', '-c', code) self.assertEqual(out, 'UTF-8 UTF-8') - out = self.get_output('-X', 'utf8', '-c', code, LC_ALL='C') - self.assertEqual(out, 'UTF-8 UTF-8') + for loc in POSIX_LOCALES: + with self.subTest(LC_ALL=loc): + out = self.get_output('-X', 'utf8', '-c', code, LC_ALL=loc) + self.assertEqual(out, 'UTF-8 UTF-8') @unittest.skipIf(MS_WINDOWS, 'test specific to Unix') def test_cmd_line(self): @@ -217,13 +222,19 @@ def check(utf8_opt, expected, **kw): self.assertEqual(args, ascii(expected), out) check('utf8', [arg_utf8]) + for loc in POSIX_LOCALES: + with self.subTest(LC_ALL=loc): + check('utf8', [arg_utf8], LC_ALL=loc) + if sys.platform == 'darwin' or support.is_android: c_arg = arg_utf8 elif sys.platform.startswith("aix"): c_arg = arg.decode('iso-8859-1') else: c_arg = arg_ascii - check('utf8=0', [c_arg], LC_ALL='C') + for loc in POSIX_LOCALES: + with self.subTest(LC_ALL=loc): + check('utf8=0', [c_arg], LC_ALL=loc) def test_optim_level(self): # CPython: check that Py_Main() doesn't increment Py_OptimizeFlag diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-08-28-11-52-13.bpo-34527.sh5MQJ.rst b/Misc/NEWS.d/next/Core and Builtins/2018-08-28-11-52-13.bpo-34527.sh5MQJ.rst new file mode 100644 index 000000000000..280a8922edc3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-08-28-11-52-13.bpo-34527.sh5MQJ.rst @@ -0,0 +1,2 @@ +The UTF-8 Mode is now also enabled by the "POSIX" locale, not only by the "C" +locale. diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-08-28-11-53-39.bpo-34527.aBEX9b.rst b/Misc/NEWS.d/next/Core and Builtins/2018-08-28-11-53-39.bpo-34527.aBEX9b.rst new file mode 100644 index 000000000000..9fce794305cc --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-08-28-11-53-39.bpo-34527.aBEX9b.rst @@ -0,0 +1,3 @@ +On FreeBSD, Py_DecodeLocale() and Py_EncodeLocale() now also forces the +ASCII encoding if the LC_CTYPE locale is "POSIX", not only if the LC_CTYPE +locale is "C". diff --git a/Python/coreconfig.c b/Python/coreconfig.c index 1b9e26e50a2b..acf46451f15a 100644 --- a/Python/coreconfig.c +++ b/Python/coreconfig.c @@ -1,5 +1,6 @@ #include "Python.h" #include "internal/pystate.h" +#include <locale.h> #define DECODE_LOCALE_ERR(NAME, LEN) \ @@ -828,14 +829,21 @@ static void config_init_locale(_PyCoreConfig *config) { if (_Py_LegacyLocaleDetected()) { - /* POSIX locale: enable C locale coercion and UTF-8 Mode */ - if (config->utf8_mode < 0) { - config->utf8_mode = 1; - } + /* The C locale enables the C locale coercion (PEP 538) */ if (config->coerce_c_locale < 0) { config->coerce_c_locale = 1; } } +#ifndef MS_WINDOWS + const char *ctype_loc = setlocale(LC_CTYPE, NULL); + if (ctype_loc != NULL + && (strcmp(ctype_loc, "C") == 0 || strcmp(ctype_loc, "POSIX") == 0)) { + /* The C locale and the POSIX locale enable the UTF-8 Mode (PEP 540) */ + if (config->utf8_mode < 0) { + config->utf8_mode = 1; + } + } +#endif } diff --git a/Python/fileutils.c b/Python/fileutils.c index 35869c81ac9f..b413f4e1e682 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -128,7 +128,7 @@ check_force_ascii(void) loc = setlocale(LC_CTYPE, NULL); if (loc == NULL) goto error; - if (strcmp(loc, "C") != 0) { + if (strcmp(loc, "C") != 0 && strcmp(loc, "POSIX") != 0) { /* the LC_CTYPE locale is different than C */ return 0; }

1 0

Daily reference leaks (4243df51fe43): sum=5
by solipsis＠pitrou.net Aug. 28, 2018

Aug. 28, 2018

results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [-7, 1, 7] memory blocks, sum=1 test_functools leaked [0, 3, 1] memory blocks, sum=4 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogLrHVgF', '--timeout', '7200']

1 0

bpo-34171: Fix test_trace. (GH-8940) (GH-8951)
by Serhiy Storchaka Aug. 28, 2018

Aug. 28, 2018

https://github.com/python/cpython/commit/e3f20828f6b7f59baa2d25437176ee32ae… commit: e3f20828f6b7f59baa2d25437176ee32ae7700b0 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: Serhiy Storchaka <storchaka(a)gmail.com> date: 2018-08-28T11:41:43+03:00 summary: bpo-34171: Fix test_trace. (GH-8940) (GH-8951) Remove "trace.cover" left from previous test runs before testing that it is no longer created. (cherry picked from commit b44a1d4f71daf4edb948b9878f82a60891e4a2e1) Co-authored-by: Serhiy Storchaka <storchaka(a)gmail.com> files: M Lib/test/test_trace.py diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py index 3b335ca42f90..63f474179d68 100644 --- a/Lib/test/test_trace.py +++ b/Lib/test/test_trace.py @@ -385,13 +385,16 @@ def tearDown(self): unlink(self.coverfile) def test_cover_files_written_no_highlight(self): + # Test also that the cover file for the trace module is not created + # (issue #34171). + tracedir = os.path.dirname(os.path.abspath(trace.__file__)) + tracecoverpath = os.path.join(tracedir, 'trace.cover') + unlink(tracecoverpath) + argv = '-m trace --count'.split() + [self.codefile] status, stdout, stderr = assert_python_ok(*argv) self.assertEqual(stderr, b'') - tracedir = os.path.dirname(os.path.abspath(trace.__file__)) - tracecoverpath = os.path.join(tracedir, "trace.cover") self.assertFalse(os.path.exists(tracecoverpath)) - self.assertTrue(os.path.exists(self.coverfile)) with open(self.coverfile) as f: self.assertEqual(f.read(),

1 0