Mailman 3 September 2023 - Python-checkins

gh-107954: Refactor initconfig.c: add CONFIG_SPEC (#110146)
by vstinner Sept. 30, 2023

Sept. 30, 2023

https://github.com/python/cpython/commit/f3bb00ea12db6525f07d62368a65efec47… commit: f3bb00ea12db6525f07d62368a65efec47d192b9 branch: main author: Victor Stinner <vstinner(a)python.org> committer: vstinner <vstinner(a)python.org> date: 2023-09-30T17:24:06Z summary: gh-107954: Refactor initconfig.c: add CONFIG_SPEC (#110146) Add a specification of the PyConfig structure to factorize the code. files: M Lib/test/test_embed.py M Python/initconfig.c M Tools/c-analyzer/cpython/ignored.tsv diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 7f1a4e665f3b5..852b3578989cd 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -455,6 +455,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'code_debug_ranges': 1, 'show_ref_count': 0, 'dump_refs': 0, + 'dump_refs_file': None, 'malloc_stats': 0, 'filesystem_encoding': GET_DEFAULT_CONFIG, diff --git a/Python/initconfig.c b/Python/initconfig.c index a0467f51d4834..089ede4623e23 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -24,6 +24,104 @@ # endif #endif +/* --- PyConfig spec ---------------------------------------------- */ + +typedef enum { + PyConfig_MEMBER_INT = 0, + PyConfig_MEMBER_UINT = 1, + PyConfig_MEMBER_ULONG = 2, + + PyConfig_MEMBER_WSTR = 10, + PyConfig_MEMBER_WSTR_OPT = 11, + PyConfig_MEMBER_WSTR_LIST = 12, +} PyConfigMemberType; + +typedef struct { + const char *name; + size_t offset; + PyConfigMemberType type; +} PyConfigSpec; + +#define SPEC(MEMBER, TYPE) \ + {#MEMBER, offsetof(PyConfig, MEMBER), PyConfig_MEMBER_##TYPE} + +static const PyConfigSpec PYCONFIG_SPEC[] = { + SPEC(_config_init, UINT), + SPEC(isolated, UINT), + SPEC(use_environment, UINT), + SPEC(dev_mode, UINT), + SPEC(install_signal_handlers, UINT), + SPEC(use_hash_seed, UINT), + SPEC(hash_seed, ULONG), + SPEC(faulthandler, UINT), + SPEC(tracemalloc, UINT), + SPEC(perf_profiling, UINT), + SPEC(import_time, UINT), + SPEC(code_debug_ranges, UINT), + SPEC(show_ref_count, UINT), + SPEC(dump_refs, UINT), + SPEC(dump_refs_file, WSTR_OPT), + SPEC(malloc_stats, UINT), + SPEC(filesystem_encoding, WSTR), + SPEC(filesystem_errors, WSTR), + SPEC(pycache_prefix, WSTR_OPT), + SPEC(parse_argv, UINT), + SPEC(orig_argv, WSTR_LIST), + SPEC(argv, WSTR_LIST), + SPEC(xoptions, WSTR_LIST), + SPEC(warnoptions, WSTR_LIST), + SPEC(site_import, UINT), + SPEC(bytes_warning, UINT), + SPEC(warn_default_encoding, UINT), + SPEC(inspect, UINT), + SPEC(interactive, UINT), + SPEC(optimization_level, UINT), + SPEC(parser_debug, UINT), + SPEC(write_bytecode, UINT), + SPEC(verbose, UINT), + SPEC(quiet, UINT), + SPEC(user_site_directory, UINT), + SPEC(configure_c_stdio, UINT), + SPEC(buffered_stdio, UINT), + SPEC(stdio_encoding, WSTR), + SPEC(stdio_errors, WSTR), +#ifdef MS_WINDOWS + SPEC(legacy_windows_stdio, UINT), +#endif + SPEC(check_hash_pycs_mode, WSTR), + SPEC(use_frozen_modules, UINT), + SPEC(safe_path, UINT), + SPEC(int_max_str_digits, INT), + SPEC(pathconfig_warnings, UINT), + SPEC(program_name, WSTR), + SPEC(pythonpath_env, WSTR_OPT), + SPEC(home, WSTR_OPT), + SPEC(platlibdir, WSTR), + SPEC(module_search_paths_set, UINT), + SPEC(module_search_paths, WSTR_LIST), + SPEC(stdlib_dir, WSTR_OPT), + SPEC(executable, WSTR_OPT), + SPEC(base_executable, WSTR_OPT), + SPEC(prefix, WSTR_OPT), + SPEC(base_prefix, WSTR_OPT), + SPEC(exec_prefix, WSTR_OPT), + SPEC(base_exec_prefix, WSTR_OPT), + SPEC(skip_source_first_line, UINT), + SPEC(run_command, WSTR_OPT), + SPEC(run_module, WSTR_OPT), + SPEC(run_filename, WSTR_OPT), + SPEC(_install_importlib, UINT), + SPEC(_init_main, UINT), + SPEC(_is_python_build, UINT), +#ifdef Py_STATS + SPEC(_pystats, UINT), +#endif + {NULL, 0, 0}, +}; + +#undef SPEC + + /* --- Command line options --------------------------------------- */ /* Short usage message (with %s for argv0) */ @@ -869,103 +967,47 @@ PyConfig_SetBytesString(PyConfig *config, wchar_t **config_str, PyStatus _PyConfig_Copy(PyConfig *config, const PyConfig *config2) { - PyStatus status; - PyConfig_Clear(config); -#define COPY_ATTR(ATTR) config->ATTR = config2->ATTR -#define COPY_WSTR_ATTR(ATTR) \ - do { \ - status = PyConfig_SetString(config, &config->ATTR, config2->ATTR); \ - if (_PyStatus_EXCEPTION(status)) { \ - return status; \ - } \ - } while (0) -#define COPY_WSTRLIST(LIST) \ - do { \ - if (_PyWideStringList_Copy(&config->LIST, &config2->LIST) < 0) { \ - return _PyStatus_NO_MEMORY(); \ - } \ - } while (0) - - COPY_ATTR(_config_init); - COPY_ATTR(isolated); - COPY_ATTR(use_environment); - COPY_ATTR(dev_mode); - COPY_ATTR(install_signal_handlers); - COPY_ATTR(use_hash_seed); - COPY_ATTR(hash_seed); - COPY_ATTR(_install_importlib); - COPY_ATTR(faulthandler); - COPY_ATTR(tracemalloc); - COPY_ATTR(perf_profiling); - COPY_ATTR(import_time); - COPY_ATTR(code_debug_ranges); - COPY_ATTR(show_ref_count); - COPY_ATTR(dump_refs); - COPY_ATTR(dump_refs_file); - COPY_ATTR(malloc_stats); - - COPY_WSTR_ATTR(pycache_prefix); - COPY_WSTR_ATTR(pythonpath_env); - COPY_WSTR_ATTR(home); - COPY_WSTR_ATTR(program_name); - - COPY_ATTR(parse_argv); - COPY_WSTRLIST(argv); - COPY_WSTRLIST(warnoptions); - COPY_WSTRLIST(xoptions); - COPY_WSTRLIST(module_search_paths); - COPY_ATTR(module_search_paths_set); - COPY_WSTR_ATTR(stdlib_dir); - - COPY_WSTR_ATTR(executable); - COPY_WSTR_ATTR(base_executable); - COPY_WSTR_ATTR(prefix); - COPY_WSTR_ATTR(base_prefix); - COPY_WSTR_ATTR(exec_prefix); - COPY_WSTR_ATTR(base_exec_prefix); - COPY_WSTR_ATTR(platlibdir); - - COPY_ATTR(site_import); - COPY_ATTR(bytes_warning); - COPY_ATTR(warn_default_encoding); - COPY_ATTR(inspect); - COPY_ATTR(interactive); - COPY_ATTR(optimization_level); - COPY_ATTR(parser_debug); - COPY_ATTR(write_bytecode); - COPY_ATTR(verbose); - COPY_ATTR(quiet); - COPY_ATTR(user_site_directory); - COPY_ATTR(configure_c_stdio); - COPY_ATTR(buffered_stdio); - COPY_WSTR_ATTR(filesystem_encoding); - COPY_WSTR_ATTR(filesystem_errors); - COPY_WSTR_ATTR(stdio_encoding); - COPY_WSTR_ATTR(stdio_errors); -#ifdef MS_WINDOWS - COPY_ATTR(legacy_windows_stdio); -#endif - COPY_ATTR(skip_source_first_line); - COPY_WSTR_ATTR(run_command); - COPY_WSTR_ATTR(run_module); - COPY_WSTR_ATTR(run_filename); - COPY_WSTR_ATTR(check_hash_pycs_mode); - COPY_ATTR(pathconfig_warnings); - COPY_ATTR(_init_main); - COPY_ATTR(use_frozen_modules); - COPY_ATTR(safe_path); - COPY_WSTRLIST(orig_argv); - COPY_ATTR(_is_python_build); - COPY_ATTR(int_max_str_digits); -#ifdef Py_STATS - COPY_ATTR(_pystats); -#endif - -#undef COPY_ATTR -#undef COPY_WSTR_ATTR -#undef COPY_WSTRLIST + PyStatus status; + const PyConfigSpec *spec = PYCONFIG_SPEC; + for (; spec->name != NULL; spec++) { + char *member = (char *)config + spec->offset; + char *member2 = (char *)config2 + spec->offset; + switch (spec->type) { + case PyConfig_MEMBER_INT: + case PyConfig_MEMBER_UINT: + { + *(int*)member = *(int*)member2; + break; + } + case PyConfig_MEMBER_ULONG: + { + *(unsigned long*)member = *(unsigned long*)member2; + break; + } + case PyConfig_MEMBER_WSTR: + case PyConfig_MEMBER_WSTR_OPT: + { + const wchar_t *str = *(const wchar_t**)member2; + status = PyConfig_SetString(config, (wchar_t**)member, str); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + break; + } + case PyConfig_MEMBER_WSTR_LIST: + { + if (_PyWideStringList_Copy((PyWideStringList*)member, + (const PyWideStringList*)member2) < 0) { + return _PyStatus_NO_MEMORY(); + } + break; + } + default: + Py_UNREACHABLE(); + } + } return _PyStatus_OK(); } @@ -978,113 +1020,58 @@ _PyConfig_AsDict(const PyConfig *config) return NULL; } -#define SET_ITEM(KEY, EXPR) \ - do { \ - PyObject *obj = (EXPR); \ - if (obj == NULL) { \ - goto fail; \ - } \ - int res = PyDict_SetItemString(dict, (KEY), obj); \ - Py_DECREF(obj); \ - if (res < 0) { \ - goto fail; \ - } \ - } while (0) -#define SET_ITEM_INT(ATTR) \ - SET_ITEM(#ATTR, PyLong_FromLong(config->ATTR)) -#define SET_ITEM_UINT(ATTR) \ - SET_ITEM(#ATTR, PyLong_FromUnsignedLong(config->ATTR)) -#define FROM_WSTRING(STR) \ - ((STR != NULL) ? \ - PyUnicode_FromWideChar(STR, -1) \ - : Py_NewRef(Py_None)) -#define SET_ITEM_WSTR(ATTR) \ - SET_ITEM(#ATTR, FROM_WSTRING(config->ATTR)) -#define SET_ITEM_WSTRLIST(LIST) \ - SET_ITEM(#LIST, _PyWideStringList_AsList(&config->LIST)) - - SET_ITEM_INT(_config_init); - SET_ITEM_INT(isolated); - SET_ITEM_INT(use_environment); - SET_ITEM_INT(dev_mode); - SET_ITEM_INT(install_signal_handlers); - SET_ITEM_INT(use_hash_seed); - SET_ITEM_UINT(hash_seed); - SET_ITEM_INT(faulthandler); - SET_ITEM_INT(tracemalloc); - SET_ITEM_INT(perf_profiling); - SET_ITEM_INT(import_time); - SET_ITEM_INT(code_debug_ranges); - SET_ITEM_INT(show_ref_count); - SET_ITEM_INT(dump_refs); - SET_ITEM_INT(malloc_stats); - SET_ITEM_WSTR(filesystem_encoding); - SET_ITEM_WSTR(filesystem_errors); - SET_ITEM_WSTR(pycache_prefix); - SET_ITEM_WSTR(program_name); - SET_ITEM_INT(parse_argv); - SET_ITEM_WSTRLIST(argv); - SET_ITEM_WSTRLIST(xoptions); - SET_ITEM_WSTRLIST(warnoptions); - SET_ITEM_WSTR(pythonpath_env); - SET_ITEM_WSTR(home); - SET_ITEM_INT(module_search_paths_set); - SET_ITEM_WSTRLIST(module_search_paths); - SET_ITEM_WSTR(stdlib_dir); - SET_ITEM_WSTR(executable); - SET_ITEM_WSTR(base_executable); - SET_ITEM_WSTR(prefix); - SET_ITEM_WSTR(base_prefix); - SET_ITEM_WSTR(exec_prefix); - SET_ITEM_WSTR(base_exec_prefix); - SET_ITEM_WSTR(platlibdir); - SET_ITEM_INT(site_import); - SET_ITEM_INT(bytes_warning); - SET_ITEM_INT(warn_default_encoding); - SET_ITEM_INT(inspect); - SET_ITEM_INT(interactive); - SET_ITEM_INT(optimization_level); - SET_ITEM_INT(parser_debug); - SET_ITEM_INT(write_bytecode); - SET_ITEM_INT(verbose); - SET_ITEM_INT(quiet); - SET_ITEM_INT(user_site_directory); - SET_ITEM_INT(configure_c_stdio); - SET_ITEM_INT(buffered_stdio); - SET_ITEM_WSTR(stdio_encoding); - SET_ITEM_WSTR(stdio_errors); -#ifdef MS_WINDOWS - SET_ITEM_INT(legacy_windows_stdio); -#endif - SET_ITEM_INT(skip_source_first_line); - SET_ITEM_WSTR(run_command); - SET_ITEM_WSTR(run_module); - SET_ITEM_WSTR(run_filename); - SET_ITEM_INT(_install_importlib); - SET_ITEM_WSTR(check_hash_pycs_mode); - SET_ITEM_INT(pathconfig_warnings); - SET_ITEM_INT(_init_main); - SET_ITEM_WSTRLIST(orig_argv); - SET_ITEM_INT(use_frozen_modules); - SET_ITEM_INT(safe_path); - SET_ITEM_INT(_is_python_build); - SET_ITEM_INT(int_max_str_digits); -#ifdef Py_STATS - SET_ITEM_INT(_pystats); -#endif + const PyConfigSpec *spec = PYCONFIG_SPEC; + for (; spec->name != NULL; spec++) { + char *member = (char *)config + spec->offset; + PyObject *obj; + switch (spec->type) { + case PyConfig_MEMBER_INT: + case PyConfig_MEMBER_UINT: + { + int value = *(int*)member; + obj = PyLong_FromLong(value); + break; + } + case PyConfig_MEMBER_ULONG: + { + unsigned long value = *(unsigned long*)member; + obj = PyLong_FromUnsignedLong(value); + break; + } + case PyConfig_MEMBER_WSTR: + case PyConfig_MEMBER_WSTR_OPT: + { + const wchar_t *wstr = *(const wchar_t**)member; + if (wstr != NULL) { + obj = PyUnicode_FromWideChar(wstr, -1); + } + else { + obj = Py_NewRef(Py_None); + } + break; + } + case PyConfig_MEMBER_WSTR_LIST: + { + const PyWideStringList *list = (const PyWideStringList*)member; + obj = _PyWideStringList_AsList(list); + break; + } + default: + Py_UNREACHABLE(); + } + if (obj == NULL) { + Py_DECREF(dict); + return NULL; + } + int res = PyDict_SetItemString(dict, spec->name, obj); + Py_DECREF(obj); + if (res < 0) { + Py_DECREF(dict); + return NULL; + } + } return dict; - -fail: - Py_DECREF(dict); - return NULL; - -#undef FROM_WSTRING -#undef SET_ITEM -#undef SET_ITEM_INT -#undef SET_ITEM_UINT -#undef SET_ITEM_WSTR -#undef SET_ITEM_WSTRLIST } @@ -1263,131 +1250,81 @@ _PyConfig_FromDict(PyConfig *config, PyObject *dict) return -1; } -#define CHECK_VALUE(NAME, TEST) \ - if (!(TEST)) { \ - config_dict_invalid_value(NAME); \ - return -1; \ + const PyConfigSpec *spec = PYCONFIG_SPEC; + for (; spec->name != NULL; spec++) { + char *member = (char *)config + spec->offset; + switch (spec->type) { + case PyConfig_MEMBER_INT: + if (config_dict_get_int(dict, spec->name, (int*)member) < 0) { + return -1; + } + break; + case PyConfig_MEMBER_UINT: + { + int value; + if (config_dict_get_int(dict, spec->name, &value) < 0) { + return -1; + } + if (value < 0) { + config_dict_invalid_value(spec->name); + return -1; + } + *(int*)member = value; + break; + } + case PyConfig_MEMBER_ULONG: + { + if (config_dict_get_ulong(dict, spec->name, + (unsigned long*)member) < 0) { + return -1; + } + break; + } + case PyConfig_MEMBER_WSTR: + { + wchar_t **wstr = (wchar_t**)member; + if (config_dict_get_wstr(dict, spec->name, config, wstr) < 0) { + return -1; + } + if (*wstr == NULL) { + config_dict_invalid_value(spec->name); + return -1; + } + break; + } + case PyConfig_MEMBER_WSTR_OPT: + { + wchar_t **wstr = (wchar_t**)member; + if (config_dict_get_wstr(dict, spec->name, config, wstr) < 0) { + return -1; + } + break; + } + case PyConfig_MEMBER_WSTR_LIST: + { + if (config_dict_get_wstrlist(dict, spec->name, config, + (PyWideStringList*)member) < 0) { + return -1; + } + break; + } + default: + Py_UNREACHABLE(); + } } -#define GET_UINT(KEY) \ - do { \ - if (config_dict_get_int(dict, #KEY, &config->KEY) < 0) { \ - return -1; \ - } \ - CHECK_VALUE(#KEY, config->KEY >= 0); \ - } while (0) -#define GET_INT(KEY) \ - do { \ - if (config_dict_get_int(dict, #KEY, &config->KEY) < 0) { \ - return -1; \ - } \ - } while (0) -#define GET_WSTR(KEY) \ - do { \ - if (config_dict_get_wstr(dict, #KEY, config, &config->KEY) < 0) { \ - return -1; \ - } \ - CHECK_VALUE(#KEY, config->KEY != NULL); \ - } while (0) -#define GET_WSTR_OPT(KEY) \ - do { \ - if (config_dict_get_wstr(dict, #KEY, config, &config->KEY) < 0) { \ - return -1; \ - } \ - } while (0) -#define GET_WSTRLIST(KEY) \ - do { \ - if (config_dict_get_wstrlist(dict, #KEY, config, &config->KEY) < 0) { \ - return -1; \ - } \ - } while (0) - GET_UINT(_config_init); - CHECK_VALUE("_config_init", - config->_config_init == _PyConfig_INIT_COMPAT - || config->_config_init == _PyConfig_INIT_PYTHON - || config->_config_init == _PyConfig_INIT_ISOLATED); - GET_UINT(isolated); - GET_UINT(use_environment); - GET_UINT(dev_mode); - GET_UINT(install_signal_handlers); - GET_UINT(use_hash_seed); - if (config_dict_get_ulong(dict, "hash_seed", &config->hash_seed) < 0) { + if (!(config->_config_init == _PyConfig_INIT_COMPAT + || config->_config_init == _PyConfig_INIT_PYTHON + || config->_config_init == _PyConfig_INIT_ISOLATED)) + { + config_dict_invalid_value("_config_init"); return -1; } - CHECK_VALUE("hash_seed", config->hash_seed <= MAX_HASH_SEED); - GET_UINT(faulthandler); - GET_UINT(tracemalloc); - GET_UINT(perf_profiling); - GET_UINT(import_time); - GET_UINT(code_debug_ranges); - GET_UINT(show_ref_count); - GET_UINT(dump_refs); - GET_UINT(malloc_stats); - GET_WSTR(filesystem_encoding); - GET_WSTR(filesystem_errors); - GET_WSTR_OPT(pycache_prefix); - GET_UINT(parse_argv); - GET_WSTRLIST(orig_argv); - GET_WSTRLIST(argv); - GET_WSTRLIST(xoptions); - GET_WSTRLIST(warnoptions); - GET_UINT(site_import); - GET_UINT(bytes_warning); - GET_UINT(warn_default_encoding); - GET_UINT(inspect); - GET_UINT(interactive); - GET_UINT(optimization_level); - GET_UINT(parser_debug); - GET_UINT(write_bytecode); - GET_UINT(verbose); - GET_UINT(quiet); - GET_UINT(user_site_directory); - GET_UINT(configure_c_stdio); - GET_UINT(buffered_stdio); - GET_WSTR(stdio_encoding); - GET_WSTR(stdio_errors); -#ifdef MS_WINDOWS - GET_UINT(legacy_windows_stdio); -#endif - GET_WSTR(check_hash_pycs_mode); - - GET_UINT(pathconfig_warnings); - GET_WSTR(program_name); - GET_WSTR_OPT(pythonpath_env); - GET_WSTR_OPT(home); - GET_WSTR(platlibdir); - - // Path configuration output - GET_UINT(module_search_paths_set); - GET_WSTRLIST(module_search_paths); - GET_WSTR_OPT(stdlib_dir); - GET_WSTR_OPT(executable); - GET_WSTR_OPT(base_executable); - GET_WSTR_OPT(prefix); - GET_WSTR_OPT(base_prefix); - GET_WSTR_OPT(exec_prefix); - GET_WSTR_OPT(base_exec_prefix); - - GET_UINT(skip_source_first_line); - GET_WSTR_OPT(run_command); - GET_WSTR_OPT(run_module); - GET_WSTR_OPT(run_filename); - - GET_UINT(_install_importlib); - GET_UINT(_init_main); - GET_UINT(use_frozen_modules); - GET_UINT(safe_path); - GET_UINT(_is_python_build); - GET_INT(int_max_str_digits); -#ifdef Py_STATS - GET_UINT(_pystats); -#endif -#undef CHECK_VALUE -#undef GET_UINT -#undef GET_INT -#undef GET_WSTR -#undef GET_WSTR_OPT + if (config->hash_seed > MAX_HASH_SEED) { + config_dict_invalid_value("hash_seed"); + return -1; + } return 0; } diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 1f398701a7a5b..c6c69a3e222f0 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -88,6 +88,10 @@ Parser/myreadline.c - PyOS_ReadlineFunctionPointer - Python/initconfig.c - _Py_StandardStreamEncoding - Python/initconfig.c - _Py_StandardStreamErrors - +# Internal constant list +Python/initconfig.c - PYCONFIG_SPEC - + + ##----------------------- ## public C-API

1 0

GH-89812: Add `pathlib._PathBase` (#106337)
by barneygale Sept. 30, 2023

Sept. 30, 2023

https://github.com/python/cpython/commit/89966a694b54f81510f06a35b1406d56a2… commit: 89966a694b54f81510f06a35b1406d56a2f2c8c5 branch: main author: Barney Gale <barney.gale(a)gmail.com> committer: barneygale <barney.gale(a)gmail.com> date: 2023-09-30T15:45:01+01:00 summary: GH-89812: Add `pathlib._PathBase` (#106337) Add private `pathlib._PathBase` class. This will be used by an experimental PyPI package to incubate a `tarfile.TarPath` class. Co-authored-by: Adam Turner <9087854+AA-Turner(a)users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2023-07-03-20-23-56.gh-issue-89812.cFkDOE.rst M Lib/pathlib.py M Lib/test/test_pathlib.py diff --git a/Lib/pathlib.py b/Lib/pathlib.py index bd5f61b0b7c87..e6be9061013a8 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -5,6 +5,7 @@ operating systems. """ +import contextlib import fnmatch import functools import io @@ -15,10 +16,19 @@ import sys import warnings from _collections_abc import Sequence -from errno import ENOENT, ENOTDIR, EBADF, ELOOP +from errno import ENOENT, ENOTDIR, EBADF, ELOOP, EINVAL from stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO from urllib.parse import quote_from_bytes as urlquote_from_bytes +try: + import pwd +except ImportError: + pwd = None +try: + import grp +except ImportError: + grp = None + __all__ = [ "UnsupportedOperation", @@ -30,6 +40,9 @@ # Internals # +# Maximum number of symlinks to follow in _PathBase.resolve() +_MAX_SYMLINKS = 40 + # Reference for Windows paths can be found at # https://learn.microsoft.com/en-gb/windows/win32/fileio/naming-a-file . _WIN_RESERVED_NAMES = frozenset( @@ -292,6 +305,11 @@ class PurePath: # The `_hash` slot stores the hash of the case-normalized string # path. It's set when `__hash__()` is called for the first time. '_hash', + + # The '_resolving' slot stores a boolean indicating whether the path + # is being processed by `_PathBase.resolve()`. This prevents duplicate + # work from occurring when `resolve()` calls `stat()` or `readlink()`. + '_resolving', ) pathmod = os.path @@ -331,6 +349,7 @@ def __init__(self, *args): f"not {type(path).__name__!r}") paths.append(path) self._raw_paths = paths + self._resolving = False def with_segments(self, *pathsegments): """Construct a new path object from any number of path-like objects. @@ -416,7 +435,7 @@ def __repr__(self): return "{}({!r})".format(self.__class__.__name__, self.as_posix()) def as_uri(self): - """Return the path as a 'file' URI.""" + """Return the path as a URI.""" if not self.is_absolute(): raise ValueError("relative path can't be expressed as a file URI") @@ -691,7 +710,9 @@ def parent(self): tail = self._tail if not tail: return self - return self._from_parsed_parts(drv, root, tail[:-1]) + path = self._from_parsed_parts(drv, root, tail[:-1]) + path._resolving = self._resolving + return path @property def parents(self): @@ -776,23 +797,35 @@ class PureWindowsPath(PurePath): # Filesystem-accessing classes -class Path(PurePath): - """PurePath subclass that can make system calls. +class _PathBase(PurePath): + """Base class for concrete path objects. - Path represents a filesystem path but unlike PurePath, also offers - methods to do system calls on path objects. Depending on your system, - instantiating a Path will return either a PosixPath or a WindowsPath - object. You can also instantiate a PosixPath or WindowsPath directly, - but cannot instantiate a WindowsPath on a POSIX system or vice versa. + This class provides dummy implementations for many methods that derived + classes can override selectively; the default implementations raise + UnsupportedOperation. The most basic methods, such as stat() and open(), + directly raise UnsupportedOperation; these basic methods are called by + other methods such as is_dir() and read_text(). + + The Path class derives this class to implement local filesystem paths. + Users may derive their own classes to implement virtual filesystem paths, + such as paths in archive files or on remote storage systems. """ __slots__ = () + __bytes__ = None + __fspath__ = None # virtual paths have no local file system representation + + def _unsupported(self, method_name): + msg = f"{type(self).__name__}.{method_name}() is unsupported" + if isinstance(self, Path): + msg += " on this system" + raise UnsupportedOperation(msg) def stat(self, *, follow_symlinks=True): """ Return the result of the stat() system call on this path, like os.stat() does. """ - return os.stat(self, follow_symlinks=follow_symlinks) + self._unsupported("stat") def lstat(self): """ @@ -859,7 +892,21 @@ def is_mount(self): """ Check if this path is a mount point """ - return os.path.ismount(self) + # Need to exist and be a dir + if not self.exists() or not self.is_dir(): + return False + + try: + parent_dev = self.parent.stat().st_dev + except OSError: + return False + + dev = self.stat().st_dev + if dev != parent_dev: + return True + ino = self.stat().st_ino + parent_ino = self.parent.stat().st_ino + return ino == parent_ino def is_symlink(self): """ @@ -880,7 +927,10 @@ def is_junction(self): """ Whether this path is a junction. """ - return os.path.isjunction(self) + # Junctions are a Windows-only feature, not present in POSIX nor the + # majority of virtual filesystems. There is no cross-platform idiom + # to check for junctions (using stat().st_mode). + return False def is_block_device(self): """ @@ -964,9 +1014,7 @@ def open(self, mode='r', buffering=-1, encoding=None, Open the file pointed by this path and return a file object, as the built-in open() function does. """ - if "b" not in mode: - encoding = io.text_encoding(encoding) - return io.open(self, mode, buffering, encoding, errors, newline) + self._unsupported("open") def read_bytes(self): """ @@ -1009,13 +1057,12 @@ def iterdir(self): The children are yielded in arbitrary order, and the special entries '.' and '..' are not included. """ - return (self._make_child_relpath(name) for name in os.listdir(self)) + self._unsupported("iterdir") def _scandir(self): - # bpo-24132: a future version of pathlib will support subclassing of - # pathlib.Path to customize how the filesystem is accessed. This - # includes scandir(), which is used to implement glob(). - return os.scandir(self) + # Emulate os.scandir(), which returns an object that can be used as a + # context manager. This method is called by walk() and glob(). + return contextlib.nullcontext(self.iterdir()) def _make_child_relpath(self, name): sep = self.pathmod.sep @@ -1144,13 +1191,13 @@ def walk(self, top_down=True, on_error=None, follow_symlinks=False): # blow up for a minor reason when (say) a thousand readable # directories are still left to visit. That logic is copied here. try: - scandir_it = path._scandir() + scandir_obj = path._scandir() except OSError as error: if on_error is not None: on_error(error) continue - with scandir_it: + with scandir_obj as scandir_it: dirnames = [] filenames = [] for entry in scandir_it: @@ -1172,17 +1219,13 @@ def walk(self, top_down=True, on_error=None, follow_symlinks=False): paths += [path._make_child_relpath(d) for d in reversed(dirnames)] - def __init__(self, *args, **kwargs): - if kwargs: - msg = ("support for supplying keyword arguments to pathlib.PurePath " - "is deprecated and scheduled for removal in Python {remove}") - warnings._deprecated("pathlib.PurePath(**kwargs)", msg, remove=(3, 14)) - super().__init__(*args) + def absolute(self): + """Return an absolute version of this path + No normalization or symlink resolution is performed. - def __new__(cls, *args, **kwargs): - if cls is Path: - cls = WindowsPath if os.name == 'nt' else PosixPath - return object.__new__(cls) + Use resolve() to resolve symlinks and remove '..' segments. + """ + self._unsupported("absolute") @classmethod def cwd(cls): @@ -1193,18 +1236,264 @@ def cwd(cls): # os.path.abspath('.') == os.getcwd(). return cls().absolute() + def expanduser(self): + """ Return a new path with expanded ~ and ~user constructs + (as returned by os.path.expanduser) + """ + self._unsupported("expanduser") + @classmethod def home(cls): - """Return a new path pointing to the user's home directory (as - returned by os.path.expanduser('~')). + """Return a new path pointing to expanduser('~'). """ return cls("~").expanduser() + def readlink(self): + """ + Return the path to which the symbolic link points. + """ + self._unsupported("readlink") + readlink._supported = False + + def _split_stack(self): + """ + Split the path into a 2-tuple (anchor, parts), where *anchor* is the + uppermost parent of the path (equivalent to path.parents[-1]), and + *parts* is a reversed list of parts following the anchor. + """ + return self._from_parsed_parts(self.drive, self.root, []), self._tail[::-1] + + def resolve(self, strict=False): + """ + Make the path absolute, resolving all symlinks on the way and also + normalizing it. + """ + if self._resolving: + return self + try: + path = self.absolute() + except UnsupportedOperation: + path = self + + # If the user has *not* overridden the `readlink()` method, then symlinks are unsupported + # and (in non-strict mode) we can improve performance by not calling `stat()`. + querying = strict or getattr(self.readlink, '_supported', True) + link_count = 0 + stat_cache = {} + target_cache = {} + path, parts = path._split_stack() + while parts: + part = parts.pop() + if part == '..': + if not path._tail: + if path.root: + # Delete '..' segment immediately following root + continue + elif path._tail[-1] != '..': + # Delete '..' segment and its predecessor + path = path.parent + continue + # Join the current part onto the path. + path_parent = path + path = path._make_child_relpath(part) + if querying and part != '..': + path._resolving = True + try: + st = stat_cache.get(path) + if st is None: + st = stat_cache[path] = path.stat(follow_symlinks=False) + if S_ISLNK(st.st_mode): + # Like Linux and macOS, raise OSError(errno.ELOOP) if too many symlinks are + # encountered during resolution. + link_count += 1 + if link_count >= _MAX_SYMLINKS: + raise OSError(ELOOP, "Too many symbolic links in path", str(path)) + target = target_cache.get(path) + if target is None: + target = target_cache[path] = path.readlink() + target, target_parts = target._split_stack() + # If the symlink target is absolute (like '/etc/hosts'), set the current + # path to its uppermost parent (like '/'). If not, the symlink target is + # relative to the symlink parent, which we recorded earlier. + path = target if target.root else path_parent + # Add the symlink target's reversed tail parts (like ['hosts', 'etc']) to + # the stack of unresolved path parts. + parts.extend(target_parts) + elif parts and not S_ISDIR(st.st_mode): + raise NotADirectoryError(ENOTDIR, "Not a directory", str(path)) + except OSError: + if strict: + raise + else: + querying = False + path._resolving = False + return path + + def symlink_to(self, target, target_is_directory=False): + """ + Make this path a symlink pointing to the target path. + Note the order of arguments (link, target) is the reverse of os.symlink. + """ + self._unsupported("symlink_to") + + def hardlink_to(self, target): + """ + Make this path a hard link pointing to the same file as *target*. + + Note the order of arguments (self, target) is the reverse of os.link's. + """ + self._unsupported("hardlink_to") + + def touch(self, mode=0o666, exist_ok=True): + """ + Create this file with the given access mode, if it doesn't exist. + """ + self._unsupported("touch") + + def mkdir(self, mode=0o777, parents=False, exist_ok=False): + """ + Create a new directory at this given path. + """ + self._unsupported("mkdir") + + def rename(self, target): + """ + Rename this path to the target path. + + The target path may be absolute or relative. Relative paths are + interpreted relative to the current working directory, *not* the + directory of the Path object. + + Returns the new Path instance pointing to the target path. + """ + self._unsupported("rename") + + def replace(self, target): + """ + Rename this path to the target path, overwriting if that path exists. + + The target path may be absolute or relative. Relative paths are + interpreted relative to the current working directory, *not* the + directory of the Path object. + + Returns the new Path instance pointing to the target path. + """ + self._unsupported("replace") + + def chmod(self, mode, *, follow_symlinks=True): + """ + Change the permissions of the path, like os.chmod(). + """ + self._unsupported("chmod") + + def lchmod(self, mode): + """ + Like chmod(), except if the path points to a symlink, the symlink's + permissions are changed, rather than its target's. + """ + self.chmod(mode, follow_symlinks=False) + + def unlink(self, missing_ok=False): + """ + Remove this file or link. + If the path is a directory, use rmdir() instead. + """ + self._unsupported("unlink") + + def rmdir(self): + """ + Remove this directory. The directory must be empty. + """ + self._unsupported("rmdir") + + def owner(self): + """ + Return the login name of the file owner. + """ + self._unsupported("owner") + + def group(self): + """ + Return the group name of the file gid. + """ + self._unsupported("group") + + def as_uri(self): + """Return the path as a URI.""" + self._unsupported("as_uri") + + +class Path(_PathBase): + """PurePath subclass that can make system calls. + + Path represents a filesystem path but unlike PurePath, also offers + methods to do system calls on path objects. Depending on your system, + instantiating a Path will return either a PosixPath or a WindowsPath + object. You can also instantiate a PosixPath or WindowsPath directly, + but cannot instantiate a WindowsPath on a POSIX system or vice versa. + """ + __slots__ = () + __bytes__ = PurePath.__bytes__ + __fspath__ = PurePath.__fspath__ + as_uri = PurePath.as_uri + + def __init__(self, *args, **kwargs): + if kwargs: + msg = ("support for supplying keyword arguments to pathlib.PurePath " + "is deprecated and scheduled for removal in Python {remove}") + warnings._deprecated("pathlib.PurePath(**kwargs)", msg, remove=(3, 14)) + super().__init__(*args) + + def __new__(cls, *args, **kwargs): + if cls is Path: + cls = WindowsPath if os.name == 'nt' else PosixPath + return object.__new__(cls) + + def stat(self, *, follow_symlinks=True): + """ + Return the result of the stat() system call on this path, like + os.stat() does. + """ + return os.stat(self, follow_symlinks=follow_symlinks) + + def is_mount(self): + """ + Check if this path is a mount point + """ + return os.path.ismount(self) + + def is_junction(self): + """ + Whether this path is a junction. + """ + return os.path.isjunction(self) + + def open(self, mode='r', buffering=-1, encoding=None, + errors=None, newline=None): + """ + Open the file pointed by this path and return a file object, as + the built-in open() function does. + """ + if "b" not in mode: + encoding = io.text_encoding(encoding) + return io.open(self, mode, buffering, encoding, errors, newline) + + def iterdir(self): + """Yield path objects of the directory contents. + + The children are yielded in arbitrary order, and the + special entries '.' and '..' are not included. + """ + return (self._make_child_relpath(name) for name in os.listdir(self)) + + def _scandir(self): + return os.scandir(self) + def absolute(self): - """Return an absolute version of this path by prepending the current - working directory. No normalization or symlink resolution is performed. + """Return an absolute version of this path + No normalization or symlink resolution is performed. - Use resolve() to get the canonical path to a file. + Use resolve() to resolve symlinks and remove '..' segments. """ if self.is_absolute(): return self @@ -1232,34 +1521,26 @@ def resolve(self, strict=False): return self.with_segments(os.path.realpath(self, strict=strict)) - def owner(self): - """ - Return the login name of the file owner. - """ - try: - import pwd + if pwd: + def owner(self): + """ + Return the login name of the file owner. + """ return pwd.getpwuid(self.stat().st_uid).pw_name - except ImportError: - raise UnsupportedOperation("Path.owner() is unsupported on this system") - - def group(self): - """ - Return the group name of the file gid. - """ - try: - import grp + if grp: + def group(self): + """ + Return the group name of the file gid. + """ return grp.getgrgid(self.stat().st_gid).gr_name - except ImportError: - raise UnsupportedOperation("Path.group() is unsupported on this system") - def readlink(self): - """ - Return the path to which the symbolic link points. - """ - if not hasattr(os, "readlink"): - raise UnsupportedOperation("os.readlink() not available on this system") - return self.with_segments(os.readlink(self)) + if hasattr(os, "readlink"): + def readlink(self): + """ + Return the path to which the symbolic link points. + """ + return self.with_segments(os.readlink(self)) def touch(self, mode=0o666, exist_ok=True): """ @@ -1306,13 +1587,6 @@ def chmod(self, mode, *, follow_symlinks=True): """ os.chmod(self, mode, follow_symlinks=follow_symlinks) - def lchmod(self, mode): - """ - Like chmod(), except if the path points to a symlink, the symlink's - permissions are changed, rather than its target's. - """ - self.chmod(mode, follow_symlinks=False) - def unlink(self, missing_ok=False): """ Remove this file or link. @@ -1356,24 +1630,22 @@ def replace(self, target): os.replace(self, target) return self.with_segments(target) - def symlink_to(self, target, target_is_directory=False): - """ - Make this path a symlink pointing to the target path. - Note the order of arguments (link, target) is the reverse of os.symlink. - """ - if not hasattr(os, "symlink"): - raise UnsupportedOperation("os.symlink() not available on this system") - os.symlink(target, self, target_is_directory) - - def hardlink_to(self, target): - """ - Make this path a hard link pointing to the same file as *target*. - - Note the order of arguments (self, target) is the reverse of os.link's. - """ - if not hasattr(os, "link"): - raise UnsupportedOperation("os.link() not available on this system") - os.link(target, self) + if hasattr(os, "symlink"): + def symlink_to(self, target, target_is_directory=False): + """ + Make this path a symlink pointing to the target path. + Note the order of arguments (link, target) is the reverse of os.symlink. + """ + os.symlink(target, self, target_is_directory) + + if hasattr(os, "link"): + def hardlink_to(self, target): + """ + Make this path a hard link pointing to the same file as *target*. + + Note the order of arguments (self, target) is the reverse of os.link's. + """ + os.link(target, self) def expanduser(self): """ Return a new path with expanded ~ and ~user constructs diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 484a5e6c3bd64..319148e9065a6 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -1582,14 +1582,172 @@ def test_group(self): # -# Tests for the concrete classes. +# Tests for the virtual classes. # -class PathTest(unittest.TestCase): - """Tests for the FS-accessing functionalities of the Path classes.""" +class PathBaseTest(PurePathTest): + cls = pathlib._PathBase - cls = pathlib.Path - can_symlink = os_helper.can_symlink() + def test_unsupported_operation(self): + P = self.cls + p = self.cls() + e = pathlib.UnsupportedOperation + self.assertRaises(e, p.stat) + self.assertRaises(e, p.lstat) + self.assertRaises(e, p.exists) + self.assertRaises(e, p.samefile, 'foo') + self.assertRaises(e, p.is_dir) + self.assertRaises(e, p.is_file) + self.assertRaises(e, p.is_mount) + self.assertRaises(e, p.is_symlink) + self.assertRaises(e, p.is_block_device) + self.assertRaises(e, p.is_char_device) + self.assertRaises(e, p.is_fifo) + self.assertRaises(e, p.is_socket) + self.assertRaises(e, p.open) + self.assertRaises(e, p.read_bytes) + self.assertRaises(e, p.read_text) + self.assertRaises(e, p.write_bytes, b'foo') + self.assertRaises(e, p.write_text, 'foo') + self.assertRaises(e, p.iterdir) + self.assertRaises(e, p.glob, '*') + self.assertRaises(e, p.rglob, '*') + self.assertRaises(e, lambda: list(p.walk())) + self.assertRaises(e, p.absolute) + self.assertRaises(e, P.cwd) + self.assertRaises(e, p.expanduser) + self.assertRaises(e, p.home) + self.assertRaises(e, p.readlink) + self.assertRaises(e, p.symlink_to, 'foo') + self.assertRaises(e, p.hardlink_to, 'foo') + self.assertRaises(e, p.mkdir) + self.assertRaises(e, p.touch) + self.assertRaises(e, p.rename, 'foo') + self.assertRaises(e, p.replace, 'foo') + self.assertRaises(e, p.chmod, 0o755) + self.assertRaises(e, p.lchmod, 0o755) + self.assertRaises(e, p.unlink) + self.assertRaises(e, p.rmdir) + self.assertRaises(e, p.owner) + self.assertRaises(e, p.group) + self.assertRaises(e, p.as_uri) + + def test_as_uri_common(self): + e = pathlib.UnsupportedOperation + self.assertRaises(e, self.cls().as_uri) + + def test_fspath_common(self): + self.assertRaises(TypeError, os.fspath, self.cls()) + + def test_as_bytes_common(self): + self.assertRaises(TypeError, bytes, self.cls()) + + def test_matches_path_api(self): + our_names = {name for name in dir(self.cls) if name[0] != '_'} + path_names = {name for name in dir(pathlib.Path) if name[0] != '_'} + self.assertEqual(our_names, path_names) + for attr_name in our_names: + our_attr = getattr(self.cls, attr_name) + path_attr = getattr(pathlib.Path, attr_name) + self.assertEqual(our_attr.__doc__, path_attr.__doc__) + + +class DummyPathIO(io.BytesIO): + """ + Used by DummyPath to implement `open('w')` + """ + + def __init__(self, files, path): + super().__init__() + self.files = files + self.path = path + + def close(self): + self.files[self.path] = self.getvalue() + super().close() + + +class DummyPath(pathlib._PathBase): + """ + Simple implementation of PathBase that keeps files and directories in + memory. + """ + _files = {} + _directories = {} + _symlinks = {} + + def stat(self, *, follow_symlinks=True): + if follow_symlinks: + path = str(self.resolve()) + else: + path = str(self.parent.resolve() / self.name) + if path in self._files: + st_mode = stat.S_IFREG + elif path in self._directories: + st_mode = stat.S_IFDIR + elif path in self._symlinks: + st_mode = stat.S_IFLNK + else: + raise FileNotFoundError(errno.ENOENT, "Not found", str(self)) + return os.stat_result((st_mode, hash(str(self)), 0, 0, 0, 0, 0, 0, 0, 0)) + + def open(self, mode='r', buffering=-1, encoding=None, + errors=None, newline=None): + if buffering != -1: + raise NotImplementedError + path_obj = self.resolve() + path = str(path_obj) + name = path_obj.name + parent = str(path_obj.parent) + if path in self._directories: + raise IsADirectoryError(errno.EISDIR, "Is a directory", path) + + text = 'b' not in mode + mode = ''.join(c for c in mode if c not in 'btU') + if mode == 'r': + if path not in self._files: + raise FileNotFoundError(errno.ENOENT, "File not found", path) + stream = io.BytesIO(self._files[path]) + elif mode == 'w': + if parent not in self._directories: + raise FileNotFoundError(errno.ENOENT, "File not found", parent) + stream = DummyPathIO(self._files, path) + self._files[path] = b'' + self._directories[parent].add(name) + else: + raise NotImplementedError + if text: + stream = io.TextIOWrapper(stream, encoding=encoding, errors=errors, newline=newline) + return stream + + def iterdir(self): + path = str(self.resolve()) + if path in self._files: + raise NotADirectoryError(errno.ENOTDIR, "Not a directory", path) + elif path in self._directories: + return (self / name for name in self._directories[path]) + else: + raise FileNotFoundError(errno.ENOENT, "File not found", path) + + def mkdir(self, mode=0o777, parents=False, exist_ok=False): + try: + self._directories[str(self.parent)].add(self.name) + self._directories[str(self)] = set() + except KeyError: + if not parents or self.parent == self: + raise FileNotFoundError(errno.ENOENT, "File not found", str(self.parent)) from None + self.parent.mkdir(parents=True, exist_ok=True) + self.mkdir(mode, parents=False, exist_ok=exist_ok) + except FileExistsError: + if not exist_ok: + raise + + +class DummyPathTest(unittest.TestCase): + """Tests for PathBase methods that use stat(), open() and iterdir().""" + + cls = DummyPath + can_symlink = False # (BASE) # | @@ -1612,37 +1770,38 @@ class PathTest(unittest.TestCase): # def setUp(self): - def cleanup(): - os.chmod(join('dirE'), 0o777) - os_helper.rmtree(BASE) - self.addCleanup(cleanup) - os.mkdir(BASE) - os.mkdir(join('dirA')) - os.mkdir(join('dirB')) - os.mkdir(join('dirC')) - os.mkdir(join('dirC', 'dirD')) - os.mkdir(join('dirE')) - with open(join('fileA'), 'wb') as f: - f.write(b"this is file A\n") - with open(join('dirB', 'fileB'), 'wb') as f: - f.write(b"this is file B\n") - with open(join('dirC', 'fileC'), 'wb') as f: - f.write(b"this is file C\n") - with open(join('dirC', 'novel.txt'), 'wb') as f: - f.write(b"this is a novel\n") - with open(join('dirC', 'dirD', 'fileD'), 'wb') as f: - f.write(b"this is file D\n") - os.chmod(join('dirE'), 0) - if self.can_symlink: - # Relative symlinks. - os.symlink('fileA', join('linkA')) - os.symlink('non-existing', join('brokenLink')) - os.symlink('dirB', join('linkB'), target_is_directory=True) - os.symlink(os.path.join('..', 'dirB'), join('dirA', 'linkC'), target_is_directory=True) - # This one goes upwards, creating a loop. - os.symlink(os.path.join('..', 'dirB'), join('dirB', 'linkD'), target_is_directory=True) - # Broken symlink (pointing to itself). - os.symlink('brokenLinkLoop', join('brokenLinkLoop')) + # note: this must be kept in sync with `PathTest.setUp()` + cls = self.cls + cls._files.clear() + cls._directories.clear() + cls._symlinks.clear() + join = cls.pathmod.join + cls._files.update({ + join(BASE, 'fileA'): b'this is file A\n', + join(BASE, 'dirB', 'fileB'): b'this is file B\n', + join(BASE, 'dirC', 'fileC'): b'this is file C\n', + join(BASE, 'dirC', 'dirD', 'fileD'): b'this is file D\n', + join(BASE, 'dirC', 'novel.txt'): b'this is a novel\n', + }) + cls._directories.update({ + BASE: {'dirA', 'dirB', 'dirC', 'dirE', 'fileA'}, + join(BASE, 'dirA'): set(), + join(BASE, 'dirB'): {'fileB'}, + join(BASE, 'dirC'): {'dirD', 'fileC', 'novel.txt'}, + join(BASE, 'dirC', 'dirD'): {'fileD'}, + join(BASE, 'dirE'): {}, + }) + dirname = BASE + while True: + dirname, basename = cls.pathmod.split(dirname) + if not basename: + break + cls._directories[dirname] = {basename} + + def tempdir(self): + path = self.cls(BASE).with_name('tmp-dirD') + path.mkdir() + return path def assertFileNotFound(self, func, *args, **kwargs): with self.assertRaises(FileNotFoundError) as cm: @@ -1991,9 +2150,11 @@ def test_rglob_symlink_loop(self): def test_glob_many_open_files(self): depth = 30 P = self.cls - base = P(BASE) / 'deep' - p = P(base, *(['d']*depth)) - p.mkdir(parents=True) + p = base = P(BASE) / 'deep' + p.mkdir() + for _ in range(depth): + p /= 'd' + p.mkdir() pattern = '/'.join(['*'] * depth) iters = [base.glob(pattern) for j in range(100)] for it in iters: @@ -2080,6 +2241,7 @@ def test_readlink(self): self.assertEqual((P / 'brokenLink').readlink(), self.cls('non-existing')) self.assertEqual((P / 'linkB').readlink(), self.cls('dirB')) + self.assertEqual((P / 'linkB' / 'linkD').readlink(), self.cls('../dirB')) with self.assertRaises(OSError): (P / 'fileA').readlink() @@ -2128,7 +2290,7 @@ def test_resolve_common(self): self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB', 'foo', 'in', 'spam'), False) p = P(BASE, 'dirA', 'linkC', '..', 'foo', 'in', 'spam') - if os.name == 'nt': + if os.name == 'nt' and isinstance(p, pathlib.Path): # In Windows, if linkY points to dirB, 'dirA\linkY\..' # resolves to 'dirA' without resolving linkY first. self._check_resolve_relative(p, P(BASE, 'dirA', 'foo', 'in', @@ -2138,9 +2300,7 @@ def test_resolve_common(self): # resolves to 'dirB/..' first before resolving to parent of dirB. self._check_resolve_relative(p, P(BASE, 'foo', 'in', 'spam'), False) # Now create absolute symlinks. - d = os_helper._longpath(tempfile.mkdtemp(suffix='-dirD', - dir=os.getcwd())) - self.addCleanup(os_helper.rmtree, d) + d = self.tempdir() P(BASE, 'dirA', 'linkX').symlink_to(d) P(BASE, str(d), 'linkY').symlink_to(join('dirB')) p = P(BASE, 'dirA', 'linkX', 'linkY', 'fileB') @@ -2150,7 +2310,7 @@ def test_resolve_common(self): self._check_resolve_relative(p, P(BASE, 'dirB', 'foo', 'in', 'spam'), False) p = P(BASE, 'dirA', 'linkX', 'linkY', '..', 'foo', 'in', 'spam') - if os.name == 'nt': + if os.name == 'nt' and isinstance(p, pathlib.Path): # In Windows, if linkY points to dirB, 'dirA\linkY\..' # resolves to 'dirA' without resolving linkY first. self._check_resolve_relative(p, P(d, 'foo', 'in', 'spam'), False) @@ -2174,6 +2334,38 @@ def test_resolve_dot(self): # Non-strict self.assertEqual(r.resolve(strict=False), p / '3' / '4') + def _check_symlink_loop(self, *args): + path = self.cls(*args) + with self.assertRaises(OSError) as cm: + path.resolve(strict=True) + self.assertEqual(cm.exception.errno, errno.ELOOP) + + def test_resolve_loop(self): + if not self.can_symlink: + self.skipTest("symlinks required") + if os.name == 'nt' and issubclass(self.cls, pathlib.Path): + self.skipTest("symlink loops work differently with concrete Windows paths") + # Loops with relative symlinks. + self.cls(BASE, 'linkX').symlink_to('linkX/inside') + self._check_symlink_loop(BASE, 'linkX') + self.cls(BASE, 'linkY').symlink_to('linkY') + self._check_symlink_loop(BASE, 'linkY') + self.cls(BASE, 'linkZ').symlink_to('linkZ/../linkZ') + self._check_symlink_loop(BASE, 'linkZ') + # Non-strict + p = self.cls(BASE, 'linkZ', 'foo') + self.assertEqual(p.resolve(strict=False), p) + # Loops with absolute symlinks. + self.cls(BASE, 'linkU').symlink_to(join('linkU/inside')) + self._check_symlink_loop(BASE, 'linkU') + self.cls(BASE, 'linkV').symlink_to(join('linkV')) + self._check_symlink_loop(BASE, 'linkV') + self.cls(BASE, 'linkW').symlink_to(join('linkW/../linkW')) + self._check_symlink_loop(BASE, 'linkW') + # Non-strict + q = self.cls(BASE, 'linkW', 'foo') + self.assertEqual(q.resolve(strict=False), q) + def test_stat(self): statA = self.cls(BASE).joinpath('fileA').stat() statB = self.cls(BASE).joinpath('dirB', 'fileB').stat() @@ -2382,6 +2574,10 @@ def _check_complex_symlinks(self, link0_target): self.assertEqualNormCase(str(p), BASE) # Resolve relative paths. + try: + self.cls().absolute() + except pathlib.UnsupportedOperation: + return old_path = os.getcwd() os.chdir(BASE) try: @@ -2409,6 +2605,92 @@ def test_complex_symlinks_relative(self): def test_complex_symlinks_relative_dot_dot(self): self._check_complex_symlinks(os.path.join('dirA', '..')) + +class DummyPathWithSymlinks(DummyPath): + def readlink(self): + path = str(self.parent.resolve() / self.name) + if path in self._symlinks: + return self.with_segments(self._symlinks[path]) + elif path in self._files or path in self._directories: + raise OSError(errno.EINVAL, "Not a symlink", path) + else: + raise FileNotFoundError(errno.ENOENT, "File not found", path) + + def symlink_to(self, target, target_is_directory=False): + self._directories[str(self.parent)].add(self.name) + self._symlinks[str(self)] = str(target) + + +class DummyPathWithSymlinksTest(DummyPathTest): + cls = DummyPathWithSymlinks + can_symlink = True + + def setUp(self): + super().setUp() + cls = self.cls + join = cls.pathmod.join + cls._symlinks.update({ + join(BASE, 'linkA'): 'fileA', + join(BASE, 'linkB'): 'dirB', + join(BASE, 'dirA', 'linkC'): join('..', 'dirB'), + join(BASE, 'dirB', 'linkD'): join('..', 'dirB'), + join(BASE, 'brokenLink'): 'non-existing', + join(BASE, 'brokenLinkLoop'): 'brokenLinkLoop', + }) + cls._directories[BASE].update({'linkA', 'linkB', 'brokenLink', 'brokenLinkLoop'}) + cls._directories[join(BASE, 'dirA')].add('linkC') + cls._directories[join(BASE, 'dirB')].add('linkD') + + +# +# Tests for the concrete classes. +# + +class PathTest(DummyPathTest): + """Tests for the FS-accessing functionalities of the Path classes.""" + cls = pathlib.Path + can_symlink = os_helper.can_symlink() + + def setUp(self): + # note: this must be kept in sync with `DummyPathTest.setUp()` + def cleanup(): + os.chmod(join('dirE'), 0o777) + os_helper.rmtree(BASE) + self.addCleanup(cleanup) + os.mkdir(BASE) + os.mkdir(join('dirA')) + os.mkdir(join('dirB')) + os.mkdir(join('dirC')) + os.mkdir(join('dirC', 'dirD')) + os.mkdir(join('dirE')) + with open(join('fileA'), 'wb') as f: + f.write(b"this is file A\n") + with open(join('dirB', 'fileB'), 'wb') as f: + f.write(b"this is file B\n") + with open(join('dirC', 'fileC'), 'wb') as f: + f.write(b"this is file C\n") + with open(join('dirC', 'novel.txt'), 'wb') as f: + f.write(b"this is a novel\n") + with open(join('dirC', 'dirD', 'fileD'), 'wb') as f: + f.write(b"this is file D\n") + os.chmod(join('dirE'), 0) + if self.can_symlink: + # Relative symlinks. + os.symlink('fileA', join('linkA')) + os.symlink('non-existing', join('brokenLink')) + os.symlink('dirB', join('linkB'), target_is_directory=True) + os.symlink(os.path.join('..', 'dirB'), join('dirA', 'linkC'), target_is_directory=True) + # This one goes upwards, creating a loop. + os.symlink(os.path.join('..', 'dirB'), join('dirB', 'linkD'), target_is_directory=True) + # Broken symlink (pointing to itself). + os.symlink('brokenLinkLoop', join('brokenLinkLoop')) + + def tempdir(self): + d = os_helper._longpath(tempfile.mkdtemp(suffix='-dirD', + dir=os.getcwd())) + self.addCleanup(os_helper.rmtree, d) + return d + def test_concrete_class(self): if self.cls is pathlib.Path: expected = pathlib.WindowsPath if os.name == 'nt' else pathlib.PosixPath @@ -3178,12 +3460,6 @@ def test_absolute(self): self.assertEqual(str(P('//a').absolute()), '//a') self.assertEqual(str(P('//a/b').absolute()), '//a/b') - def _check_symlink_loop(self, *args): - path = self.cls(*args) - with self.assertRaises(OSError) as cm: - path.resolve(strict=True) - self.assertEqual(cm.exception.errno, errno.ELOOP) - @unittest.skipIf( is_emscripten or is_wasi, "umask is not implemented on Emscripten/WASI." @@ -3230,30 +3506,6 @@ def test_touch_mode(self): st = os.stat(join('masked_new_file')) self.assertEqual(stat.S_IMODE(st.st_mode), 0o750) - def test_resolve_loop(self): - if not self.can_symlink: - self.skipTest("symlinks required") - # Loops with relative symlinks. - os.symlink('linkX/inside', join('linkX')) - self._check_symlink_loop(BASE, 'linkX') - os.symlink('linkY', join('linkY')) - self._check_symlink_loop(BASE, 'linkY') - os.symlink('linkZ/../linkZ', join('linkZ')) - self._check_symlink_loop(BASE, 'linkZ') - # Non-strict - p = self.cls(BASE, 'linkZ', 'foo') - self.assertEqual(p.resolve(strict=False), p) - # Loops with absolute symlinks. - os.symlink(join('linkU/inside'), join('linkU')) - self._check_symlink_loop(BASE, 'linkU') - os.symlink(join('linkV'), join('linkV')) - self._check_symlink_loop(BASE, 'linkV') - os.symlink(join('linkW/../linkW'), join('linkW')) - self._check_symlink_loop(BASE, 'linkW') - # Non-strict - q = self.cls(BASE, 'linkW', 'foo') - self.assertEqual(q.resolve(strict=False), q) - def test_glob(self): P = self.cls p = P(BASE) diff --git a/Misc/NEWS.d/next/Library/2023-07-03-20-23-56.gh-issue-89812.cFkDOE.rst b/Misc/NEWS.d/next/Library/2023-07-03-20-23-56.gh-issue-89812.cFkDOE.rst new file mode 100644 index 0000000000000..a4221fc4ca900 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-03-20-23-56.gh-issue-89812.cFkDOE.rst @@ -0,0 +1,2 @@ +Add private ``pathlib._PathBase`` class, which provides experimental support +for virtual filesystems, and may be made public in a future version of Python.

1 0

[3.11] GH-101100: Fix reference warnings for ``gettext`` (GH-110115) (#110141)
by hugovk Sept. 30, 2023

Sept. 30, 2023

https://github.com/python/cpython/commit/cb1f49991e86d773f6d36a49b81be12abf… commit: cb1f49991e86d773f6d36a49b81be12abf811057 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: hugovk <hugovk(a)users.noreply.github.com> date: 2023-09-30T11:21:27Z summary: [3.11] GH-101100: Fix reference warnings for ``gettext`` (GH-110115) (#110141) Co-authored-by: Adam Turner <9087854+AA-Turner(a)users.noreply.github.com> files: M Doc/library/gettext.rst diff --git a/Doc/library/gettext.rst b/Doc/library/gettext.rst index 88a65b980d310..7ebe91b372d35 100644 --- a/Doc/library/gettext.rst +++ b/Doc/library/gettext.rst @@ -58,7 +58,7 @@ class-based API instead. Return the localized translation of *message*, based on the current global domain, language, and locale directory. This function is usually aliased as - :func:`_` in the local namespace (see examples below). + :func:`!_` in the local namespace (see examples below). .. function:: dgettext(domain, message) @@ -98,7 +98,7 @@ class-based API instead. .. versionadded:: 3.8 -Note that GNU :program:`gettext` also defines a :func:`dcgettext` method, but +Note that GNU :program:`gettext` also defines a :func:`!dcgettext` method, but this was deemed not useful and so it is currently unimplemented. Here's an example of typical usage for this API:: @@ -119,7 +119,7 @@ greater convenience than the GNU :program:`gettext` API. It is the recommended way of localizing your Python applications and modules. :mod:`!gettext` defines a :class:`GNUTranslations` class which implements the parsing of GNU :file:`.mo` format files, and has methods for returning strings. Instances of this class can also -install themselves in the built-in namespace as the function :func:`_`. +install themselves in the built-in namespace as the function :func:`!_`. .. function:: find(domain, localedir=None, languages=None, all=False) @@ -150,15 +150,12 @@ install themselves in the built-in namespace as the function :func:`_`. .. function:: translation(domain, localedir=None, languages=None, class_=None, fallback=False) - Return a :class:`*Translations` instance based on the *domain*, *localedir*, + Return a ``*Translations`` instance based on the *domain*, *localedir*, and *languages*, which are first passed to :func:`find` to get a list of the associated :file:`.mo` file paths. Instances with identical :file:`.mo` file names are cached. The actual class instantiated is *class_* if provided, otherwise :class:`GNUTranslations`. The class's constructor must - take a single :term:`file object` argument. If provided, *codeset* will change - the charset used to encode translated strings in the - :meth:`~NullTranslations.lgettext` and :meth:`~NullTranslations.lngettext` - methods. + take a single :term:`file object` argument. If multiple files are found, later files are used as fallbacks for earlier ones. To allow setting the fallback, :func:`copy.copy` is used to clone each @@ -177,19 +174,19 @@ install themselves in the built-in namespace as the function :func:`_`. .. function:: install(domain, localedir=None, *, names=None) - This installs the function :func:`_` in Python's builtins namespace, based on + This installs the function :func:`!_` in Python's builtins namespace, based on *domain* and *localedir* which are passed to the function :func:`translation`. For the *names* parameter, please see the description of the translation object's :meth:`~NullTranslations.install` method. As seen below, you usually mark the strings in your application that are - candidates for translation, by wrapping them in a call to the :func:`_` + candidates for translation, by wrapping them in a call to the :func:`!_` function, like this:: print(_('This string will be translated.')) - For convenience, you want the :func:`_` function to be installed in Python's + For convenience, you want the :func:`!_` function to be installed in Python's builtins namespace, so it is easily accessible in all modules of your application. @@ -276,20 +273,20 @@ are the methods of :class:`!NullTranslations`: If the *names* parameter is given, it must be a sequence containing the names of functions you want to install in the builtins namespace in - addition to :func:`_`. Supported names are ``'gettext'``, ``'ngettext'``, - ``'pgettext'``, ``'npgettext'``, ``'lgettext'``, and ``'lngettext'``. + addition to :func:`!_`. Supported names are ``'gettext'``, ``'ngettext'``, + ``'pgettext'``, and ``'npgettext'``. Note that this is only one way, albeit the most convenient way, to make - the :func:`_` function available to your application. Because it affects + the :func:`!_` function available to your application. Because it affects the entire application globally, and specifically the built-in namespace, - localized modules should never install :func:`_`. Instead, they should use - this code to make :func:`_` available to their module:: + localized modules should never install :func:`!_`. Instead, they should use + this code to make :func:`!_` available to their module:: import gettext t = gettext.translation('mymodule', ...) _ = t.gettext - This puts :func:`_` only in the module's global namespace and so only + This puts :func:`!_` only in the module's global namespace and so only affects calls within this module. .. versionchanged:: 3.8 @@ -314,7 +311,7 @@ initialize the "protected" :attr:`_charset` instance variable, defaulting to ids and message strings read from the catalog are converted to Unicode using this encoding, else ASCII is assumed. -Since message ids are read as Unicode strings too, all :meth:`*gettext` methods +Since message ids are read as Unicode strings too, all ``*gettext()`` methods will assume message ids as Unicode strings, not byte strings. The entire set of key/value pairs are placed into a dictionary and set as the @@ -404,7 +401,7 @@ version has a slightly different API. Its documented usage was:: _ = cat.gettext print(_('hello world')) -For compatibility with this older module, the function :func:`Catalog` is an +For compatibility with this older module, the function :func:`!Catalog` is an alias for the :func:`translation` function described above. One difference between this module and Henstridge's: his catalog objects @@ -432,7 +429,7 @@ take the following steps: In order to prepare your code for I18N, you need to look at all the strings in your files. Any string that needs to be translated should be marked by wrapping -it in ``_('...')`` --- that is, a call to the function :func:`_`. For example:: +it in ``_('...')`` --- that is, a call to the function :func:`_ <gettext>`. For example:: filename = 'mylog.txt' message = _('writing a log message') @@ -504,7 +501,7 @@ module:: Localizing your application ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -If you are localizing your application, you can install the :func:`_` function +If you are localizing your application, you can install the :func:`!_` function globally into the built-in namespace, usually in the main driver file of your application. This will let all your application-specific files just use ``_('...')`` without having to explicitly install it in each file. @@ -581,13 +578,13 @@ Here is one way you can handle this situation:: for a in animals: print(_(a)) -This works because the dummy definition of :func:`_` simply returns the string +This works because the dummy definition of :func:`!_` simply returns the string unchanged. And this dummy definition will temporarily override any definition -of :func:`_` in the built-in namespace (until the :keyword:`del` command). Take -care, though if you have a previous definition of :func:`_` in the local +of :func:`!_` in the built-in namespace (until the :keyword:`del` command). Take +care, though if you have a previous definition of :func:`!_` in the local namespace. -Note that the second use of :func:`_` will not identify "a" as being +Note that the second use of :func:`!_` will not identify "a" as being translatable to the :program:`gettext` program, because the parameter is not a string literal. @@ -606,13 +603,13 @@ Another way to handle this is with the following example:: print(_(a)) In this case, you are marking translatable strings with the function -:func:`N_`, which won't conflict with any definition of :func:`_`. +:func:`!N_`, which won't conflict with any definition of :func:`!_`. However, you will need to teach your message extraction program to -look for translatable strings marked with :func:`N_`. :program:`xgettext`, +look for translatable strings marked with :func:`!N_`. :program:`xgettext`, :program:`pygettext`, ``pybabel extract``, and :program:`xpot` all support this through the use of the :option:`!-k` command-line switch. -The choice of :func:`N_` here is totally arbitrary; it could have just -as easily been :func:`MarkThisStringForTranslation`. +The choice of :func:`!N_` here is totally arbitrary; it could have just +as easily been :func:`!MarkThisStringForTranslation`. Acknowledgements

1 0

GH-101100: Fix reference warnings for ``gettext`` (#110115)
by hugovk Sept. 30, 2023

Sept. 30, 2023

https://github.com/python/cpython/commit/0449fe999d56ba795a852d83380fe06514… commit: 0449fe999d56ba795a852d83380fe06514139935 branch: main author: Adam Turner <9087854+AA-Turner(a)users.noreply.github.com> committer: hugovk <hugovk(a)users.noreply.github.com> date: 2023-09-30T05:10:07-06:00 summary: GH-101100: Fix reference warnings for ``gettext`` (#110115) files: M Doc/library/gettext.rst diff --git a/Doc/library/gettext.rst b/Doc/library/gettext.rst index 88a65b980d310..7ebe91b372d35 100644 --- a/Doc/library/gettext.rst +++ b/Doc/library/gettext.rst @@ -58,7 +58,7 @@ class-based API instead. Return the localized translation of *message*, based on the current global domain, language, and locale directory. This function is usually aliased as - :func:`_` in the local namespace (see examples below). + :func:`!_` in the local namespace (see examples below). .. function:: dgettext(domain, message) @@ -98,7 +98,7 @@ class-based API instead. .. versionadded:: 3.8 -Note that GNU :program:`gettext` also defines a :func:`dcgettext` method, but +Note that GNU :program:`gettext` also defines a :func:`!dcgettext` method, but this was deemed not useful and so it is currently unimplemented. Here's an example of typical usage for this API:: @@ -119,7 +119,7 @@ greater convenience than the GNU :program:`gettext` API. It is the recommended way of localizing your Python applications and modules. :mod:`!gettext` defines a :class:`GNUTranslations` class which implements the parsing of GNU :file:`.mo` format files, and has methods for returning strings. Instances of this class can also -install themselves in the built-in namespace as the function :func:`_`. +install themselves in the built-in namespace as the function :func:`!_`. .. function:: find(domain, localedir=None, languages=None, all=False) @@ -150,15 +150,12 @@ install themselves in the built-in namespace as the function :func:`_`. .. function:: translation(domain, localedir=None, languages=None, class_=None, fallback=False) - Return a :class:`*Translations` instance based on the *domain*, *localedir*, + Return a ``*Translations`` instance based on the *domain*, *localedir*, and *languages*, which are first passed to :func:`find` to get a list of the associated :file:`.mo` file paths. Instances with identical :file:`.mo` file names are cached. The actual class instantiated is *class_* if provided, otherwise :class:`GNUTranslations`. The class's constructor must - take a single :term:`file object` argument. If provided, *codeset* will change - the charset used to encode translated strings in the - :meth:`~NullTranslations.lgettext` and :meth:`~NullTranslations.lngettext` - methods. + take a single :term:`file object` argument. If multiple files are found, later files are used as fallbacks for earlier ones. To allow setting the fallback, :func:`copy.copy` is used to clone each @@ -177,19 +174,19 @@ install themselves in the built-in namespace as the function :func:`_`. .. function:: install(domain, localedir=None, *, names=None) - This installs the function :func:`_` in Python's builtins namespace, based on + This installs the function :func:`!_` in Python's builtins namespace, based on *domain* and *localedir* which are passed to the function :func:`translation`. For the *names* parameter, please see the description of the translation object's :meth:`~NullTranslations.install` method. As seen below, you usually mark the strings in your application that are - candidates for translation, by wrapping them in a call to the :func:`_` + candidates for translation, by wrapping them in a call to the :func:`!_` function, like this:: print(_('This string will be translated.')) - For convenience, you want the :func:`_` function to be installed in Python's + For convenience, you want the :func:`!_` function to be installed in Python's builtins namespace, so it is easily accessible in all modules of your application. @@ -276,20 +273,20 @@ are the methods of :class:`!NullTranslations`: If the *names* parameter is given, it must be a sequence containing the names of functions you want to install in the builtins namespace in - addition to :func:`_`. Supported names are ``'gettext'``, ``'ngettext'``, - ``'pgettext'``, ``'npgettext'``, ``'lgettext'``, and ``'lngettext'``. + addition to :func:`!_`. Supported names are ``'gettext'``, ``'ngettext'``, + ``'pgettext'``, and ``'npgettext'``. Note that this is only one way, albeit the most convenient way, to make - the :func:`_` function available to your application. Because it affects + the :func:`!_` function available to your application. Because it affects the entire application globally, and specifically the built-in namespace, - localized modules should never install :func:`_`. Instead, they should use - this code to make :func:`_` available to their module:: + localized modules should never install :func:`!_`. Instead, they should use + this code to make :func:`!_` available to their module:: import gettext t = gettext.translation('mymodule', ...) _ = t.gettext - This puts :func:`_` only in the module's global namespace and so only + This puts :func:`!_` only in the module's global namespace and so only affects calls within this module. .. versionchanged:: 3.8 @@ -314,7 +311,7 @@ initialize the "protected" :attr:`_charset` instance variable, defaulting to ids and message strings read from the catalog are converted to Unicode using this encoding, else ASCII is assumed. -Since message ids are read as Unicode strings too, all :meth:`*gettext` methods +Since message ids are read as Unicode strings too, all ``*gettext()`` methods will assume message ids as Unicode strings, not byte strings. The entire set of key/value pairs are placed into a dictionary and set as the @@ -404,7 +401,7 @@ version has a slightly different API. Its documented usage was:: _ = cat.gettext print(_('hello world')) -For compatibility with this older module, the function :func:`Catalog` is an +For compatibility with this older module, the function :func:`!Catalog` is an alias for the :func:`translation` function described above. One difference between this module and Henstridge's: his catalog objects @@ -432,7 +429,7 @@ take the following steps: In order to prepare your code for I18N, you need to look at all the strings in your files. Any string that needs to be translated should be marked by wrapping -it in ``_('...')`` --- that is, a call to the function :func:`_`. For example:: +it in ``_('...')`` --- that is, a call to the function :func:`_ <gettext>`. For example:: filename = 'mylog.txt' message = _('writing a log message') @@ -504,7 +501,7 @@ module:: Localizing your application ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -If you are localizing your application, you can install the :func:`_` function +If you are localizing your application, you can install the :func:`!_` function globally into the built-in namespace, usually in the main driver file of your application. This will let all your application-specific files just use ``_('...')`` without having to explicitly install it in each file. @@ -581,13 +578,13 @@ Here is one way you can handle this situation:: for a in animals: print(_(a)) -This works because the dummy definition of :func:`_` simply returns the string +This works because the dummy definition of :func:`!_` simply returns the string unchanged. And this dummy definition will temporarily override any definition -of :func:`_` in the built-in namespace (until the :keyword:`del` command). Take -care, though if you have a previous definition of :func:`_` in the local +of :func:`!_` in the built-in namespace (until the :keyword:`del` command). Take +care, though if you have a previous definition of :func:`!_` in the local namespace. -Note that the second use of :func:`_` will not identify "a" as being +Note that the second use of :func:`!_` will not identify "a" as being translatable to the :program:`gettext` program, because the parameter is not a string literal. @@ -606,13 +603,13 @@ Another way to handle this is with the following example:: print(_(a)) In this case, you are marking translatable strings with the function -:func:`N_`, which won't conflict with any definition of :func:`_`. +:func:`!N_`, which won't conflict with any definition of :func:`!_`. However, you will need to teach your message extraction program to -look for translatable strings marked with :func:`N_`. :program:`xgettext`, +look for translatable strings marked with :func:`!N_`. :program:`xgettext`, :program:`pygettext`, ``pybabel extract``, and :program:`xpot` all support this through the use of the :option:`!-k` command-line switch. -The choice of :func:`N_` here is totally arbitrary; it could have just -as easily been :func:`MarkThisStringForTranslation`. +The choice of :func:`!N_` here is totally arbitrary; it could have just +as easily been :func:`!MarkThisStringForTranslation`. Acknowledgements

1 0

[3.11] GH-101100: Fix reference warnings for ``namedtuple`` (GH-110113) (#110136)
by JelleZijlstra Sept. 30, 2023

Sept. 30, 2023

https://github.com/python/cpython/commit/a673248d6cb1ae5d5b7fbf0ab6c8db8c32… commit: a673248d6cb1ae5d5b7fbf0ab6c8db8c32df0eee branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: JelleZijlstra <jelle.zijlstra(a)gmail.com> date: 2023-09-30T06:42:37Z summary: [3.11] GH-101100: Fix reference warnings for ``namedtuple`` (GH-110113) (#110136) GH-101100: Fix reference warnings for ``namedtuple`` (GH-110113) (cherry picked from commit cbdacc738a52a876aae5b74b4665d30a5f204766) Co-authored-by: Adam Turner <9087854+AA-Turner(a)users.noreply.github.com> files: M Doc/whatsnew/2.6.rst M Misc/NEWS.d/3.8.0a1.rst diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index 128407e3fba13..96d9b792b3723 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -1850,8 +1850,8 @@ changes, or look through the Subversion logs for all the details. special values and floating-point exceptions in a manner consistent with Annex 'G' of the C99 standard. -* A new data type in the :mod:`collections` module: :class:`namedtuple(typename, - fieldnames)` is a factory function that creates subclasses of the standard tuple +* A new data type in the :mod:`collections` module: ``namedtuple(typename, fieldnames)`` + is a factory function that creates subclasses of the standard tuple whose fields are accessible by name as well as index. For example:: >>> var_type = collections.namedtuple('variable', @@ -1873,7 +1873,7 @@ changes, or look through the Subversion logs for all the details. variable(id=1, name='amplitude', type='int', size=4) Several places in the standard library that returned tuples have - been modified to return :class:`namedtuple` instances. For example, + been modified to return :func:`namedtuple` instances. For example, the :meth:`Decimal.as_tuple` method now returns a named tuple with :attr:`sign`, :attr:`digits`, and :attr:`exponent` fields. diff --git a/Misc/NEWS.d/3.8.0a1.rst b/Misc/NEWS.d/3.8.0a1.rst index 530260aba873c..4adacfd41809d 100644 --- a/Misc/NEWS.d/3.8.0a1.rst +++ b/Misc/NEWS.d/3.8.0a1.rst @@ -380,7 +380,7 @@ Implement :pep:`572` (assignment expressions). Patch by Emily Morehouse. .. nonce: voIdcp .. section: Core and Builtins -Speed up :class:`namedtuple` attribute access by 1.6x using a C fast-path +Speed up :func:`namedtuple` attribute access by 1.6x using a C fast-path for the name descriptors. Patch by Pablo Galindo. ..

1 0

GH-101100: Fix reference warnings for ``namedtuple`` (#110113)
by JelleZijlstra Sept. 30, 2023

Sept. 30, 2023

https://github.com/python/cpython/commit/cbdacc738a52a876aae5b74b4665d30a5f… commit: cbdacc738a52a876aae5b74b4665d30a5f204766 branch: main author: Adam Turner <9087854+AA-Turner(a)users.noreply.github.com> committer: JelleZijlstra <jelle.zijlstra(a)gmail.com> date: 2023-09-29T23:32:35-07:00 summary: GH-101100: Fix reference warnings for ``namedtuple`` (#110113) files: M Doc/whatsnew/2.6.rst M Misc/NEWS.d/3.8.0a1.rst diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index f3912d42180bf..2f749dc40f7ee 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -1850,8 +1850,8 @@ changes, or look through the Subversion logs for all the details. special values and floating-point exceptions in a manner consistent with Annex 'G' of the C99 standard. -* A new data type in the :mod:`collections` module: :class:`namedtuple(typename, - fieldnames)` is a factory function that creates subclasses of the standard tuple +* A new data type in the :mod:`collections` module: ``namedtuple(typename, fieldnames)`` + is a factory function that creates subclasses of the standard tuple whose fields are accessible by name as well as index. For example:: >>> var_type = collections.namedtuple('variable', @@ -1873,7 +1873,7 @@ changes, or look through the Subversion logs for all the details. variable(id=1, name='amplitude', type='int', size=4) Several places in the standard library that returned tuples have - been modified to return :class:`namedtuple` instances. For example, + been modified to return :func:`namedtuple` instances. For example, the :meth:`Decimal.as_tuple` method now returns a named tuple with :attr:`sign`, :attr:`digits`, and :attr:`exponent` fields. diff --git a/Misc/NEWS.d/3.8.0a1.rst b/Misc/NEWS.d/3.8.0a1.rst index 3cbbbf7465032..0dc6e945719ec 100644 --- a/Misc/NEWS.d/3.8.0a1.rst +++ b/Misc/NEWS.d/3.8.0a1.rst @@ -380,7 +380,7 @@ Implement :pep:`572` (assignment expressions). Patch by Emily Morehouse. .. nonce: voIdcp .. section: Core and Builtins -Speed up :class:`namedtuple` attribute access by 1.6x using a C fast-path +Speed up :func:`namedtuple` attribute access by 1.6x using a C fast-path for the name descriptors. Patch by Pablo Galindo. ..

1 0

Add example for linear_regression() with proportional=True. (gh-110133)
by rhettinger Sept. 30, 2023

Sept. 30, 2023

https://github.com/python/cpython/commit/613c0d4e866341e15a66704643a6392ce4… commit: 613c0d4e866341e15a66704643a6392ce49058ba branch: main author: Raymond Hettinger <rhettinger(a)users.noreply.github.com> committer: rhettinger <rhettinger(a)users.noreply.github.com> date: 2023-09-29T23:18:12-05:00 summary: Add example for linear_regression() with proportional=True. (gh-110133) files: M Doc/library/statistics.rst diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index a8a7901256532..f3c1bf20ae3ac 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -14,6 +14,7 @@ .. testsetup:: * from statistics import * + import math __name__ = '<doctest>' -------------- @@ -741,6 +742,24 @@ However, for reading convenience, most of the examples show sorted sequences. *y = slope \* x + noise* + Continuing the example from :func:`correlation`, we look to see + how well a model based on major planets can predict the orbital + distances for dwarf planets: + + .. doctest:: + + >>> model = linear_regression(period_squared, dist_cubed, proportional=True) + >>> slope = model.slope + + >>> # Dwarf planets: Pluto, Eris, Makemake, Haumea, Ceres + >>> orbital_periods = [90_560, 204_199, 111_845, 103_410, 1_680] # days + >>> predicted_dist = [math.cbrt(slope * (p * p)) for p in orbital_periods] + >>> list(map(round, predicted_dist)) + [5912, 10166, 6806, 6459, 414] + + >>> [5_906, 10_152, 6_796, 6_450, 414] # actual distance in million km + [5906, 10152, 6796, 6450, 414] + .. versionadded:: 3.10 .. versionchanged:: 3.11

1 0

[3.11] gh-107888: Fix test_mmap PROT_EXEC comment (GH-110125) (#110130)
by vstinner Sept. 29, 2023

Sept. 29, 2023

https://github.com/python/cpython/commit/42b6883d5f648a95e33f815364c8a529e4… commit: 42b6883d5f648a95e33f815364c8a529e4574caa branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: vstinner <vstinner(a)python.org> date: 2023-09-29T22:19:33Z summary: [3.11] gh-107888: Fix test_mmap PROT_EXEC comment (GH-110125) (#110130) gh-107888: Fix test_mmap PROT_EXEC comment (GH-110125) (cherry picked from commit 14098b78f7453adbd40c53e32c29588611b7c87b) Co-authored-by: Victor Stinner <vstinner(a)python.org> files: M Lib/test/test_mmap.py diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index 92c99d645b25c..dfcf3039422af 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -258,7 +258,7 @@ def test_access_parameter(self): try: m = mmap.mmap(f.fileno(), mapsize, prot=prot) except PermissionError: - # on macOS 14, PROT_READ | PROT_WRITE is not allowed + # on macOS 14, PROT_READ | PROT_EXEC is not allowed pass else: self.assertRaises(TypeError, m.write, b"abcdef")

1 0

gh-107888: Fix test_mmap PROT_EXEC comment (#110125)
by vstinner Sept. 29, 2023

Sept. 29, 2023

https://github.com/python/cpython/commit/14098b78f7453adbd40c53e32c29588611… commit: 14098b78f7453adbd40c53e32c29588611b7c87b branch: main author: Victor Stinner <vstinner(a)python.org> committer: vstinner <vstinner(a)python.org> date: 2023-09-29T23:56:19+02:00 summary: gh-107888: Fix test_mmap PROT_EXEC comment (#110125) files: M Lib/test/test_mmap.py diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index 92c99d645b25c..dfcf3039422af 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -258,7 +258,7 @@ def test_access_parameter(self): try: m = mmap.mmap(f.fileno(), mapsize, prot=prot) except PermissionError: - # on macOS 14, PROT_READ | PROT_WRITE is not allowed + # on macOS 14, PROT_READ | PROT_EXEC is not allowed pass else: self.assertRaises(TypeError, m.write, b"abcdef")

1 0

[3.11] gh-108851: Fix support.get_recursion_available() for USE_STACKCHECK (#110127)
by vstinner Sept. 29, 2023

Sept. 29, 2023

https://github.com/python/cpython/commit/190e8fbfb7284e9c253388f0c2363cd838… commit: 190e8fbfb7284e9c253388f0c2363cd8387e6e7f branch: 3.11 author: Victor Stinner <vstinner(a)python.org> committer: vstinner <vstinner(a)python.org> date: 2023-09-29T23:54:46+02:00 summary: [3.11] gh-108851: Fix support.get_recursion_available() for USE_STACKCHECK (#110127) Add _testcapi.USE_STACKCHECK. USE_STACKCHECK on using on Windows 32-bit. files: M Lib/test/support/__init__.py M Lib/test/test_support.py M Modules/_testcapimodule.c diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 2c6b22fdee5a2..2e6518cf241f1 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2243,7 +2243,16 @@ def get_recursion_available(): """ limit = sys.getrecursionlimit() depth = get_recursion_depth() - return limit - depth + + try: + from _testcapi import USE_STACKCHECK + except ImportError: + USE_STACKCHECK = False + + if USE_STACKCHECK: + return max(limit - depth - 1, 0) + else: + return limit - depth @contextlib.contextmanager def set_recursion_limit(limit): diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 9894c2647d7c9..2efdbd22d90e2 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -704,6 +704,10 @@ def test_get_recursion_depth(self): code = textwrap.dedent(""" from test import support import sys + try: + from _testcapi import USE_STACKCHECK + except ImportError: + USE_STACKCHECK = False def check(cond): if not cond: @@ -728,19 +732,24 @@ def test_recursive(depth, limit): check(get_depth == depth) test_recursive(depth + 1, limit) + if USE_STACKCHECK: + # f-string consumes 2 frames and -1 for USE_STACKCHECK + IGNORE = 3 + else: + # f-string consumes 2 frames + IGNORE = 2 + # depth up to 25 with support.infinite_recursion(max_depth=25): limit = sys.getrecursionlimit() print(f"test with sys.getrecursionlimit()={limit}") - # Use limit-2 since f-string seems to consume 2 frames. - test_recursive(2, limit - 2) + test_recursive(2, limit - IGNORE) # depth up to 500 with support.infinite_recursion(max_depth=500): limit = sys.getrecursionlimit() print(f"test with sys.getrecursionlimit()={limit}") - # limit-2 since f-string seems to consume 2 frames - test_recursive(2, limit - 2) + test_recursive(2, limit - IGNORE) """) script_helper.assert_python_ok("-c", code) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 5c00b48001a91..2f1801f781017 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -8203,8 +8203,14 @@ PyInit__testcapi(void) #else v = Py_False; #endif - Py_INCREF(v); - PyModule_AddObject(m, "WITH_PYMALLOC", v); + PyModule_AddObject(m, "WITH_PYMALLOC", Py_NewRef(v)); + +#ifdef USE_STACKCHECK + v = Py_True; +#else + v = Py_False; +#endif + PyModule_AddObject(m, "USE_STACKCHECK", Py_NewRef(v)); TestError = PyErr_NewException("_testcapi.error", NULL, NULL); Py_INCREF(TestError);

1 0