Mailman 3 November 2024 - Python-checkins

[3.11] gh-124651: Quote template strings in `venv` activation scripts (GH-124712) (GH-126185) (#126269)
by pablogsal Nov. 1, 2024

Nov. 1, 2024

https://github.com/python/cpython/commit/ae961ae94bf19c8f8c7fbea3d1c25cc55c… commit: ae961ae94bf19c8f8c7fbea3d1c25cc55ce8ae97 branch: 3.11 author: Victor Stinner <vstinner(a)python.org> committer: pablogsal <Pablogsal(a)gmail.com> date: 2024-11-01T13:11:47Z summary: [3.11] gh-124651: Quote template strings in `venv` activation scripts (GH-124712) (GH-126185) (#126269) files: A Misc/NEWS.d/next/Library/2024-09-28-02-03-04.gh-issue-124651.bLBGtH.rst M Lib/test/test_venv.py M Lib/venv/__init__.py M Lib/venv/scripts/common/activate M Lib/venv/scripts/nt/activate.bat M Lib/venv/scripts/posix/activate.csh M Lib/venv/scripts/posix/activate.fish diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 9563282e6d9b9c..d3abb77f40f35f 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -17,7 +17,8 @@ import sys import sysconfig import tempfile -from test.support import (captured_stdout, captured_stderr, requires_zlib, +import shlex +from test.support import (captured_stdout, captured_stderr, skip_if_broken_multiprocessing_synchronize, verbose, requires_subprocess, is_emscripten, is_wasi, requires_venv_with_pip, TEST_HOME_DIR, @@ -96,6 +97,10 @@ def get_text_file_contents(self, *args, encoding='utf-8'): result = f.read() return result + def assertEndsWith(self, string, tail): + if not string.endswith(tail): + self.fail(f"String {string!r} does not end with {tail!r}") + class BasicTest(BaseTest): """Test venv module functionality.""" @@ -446,6 +451,82 @@ def test_executable_symlinks(self): 'import sys; print(sys.executable)']) self.assertEqual(out.strip(), envpy.encode()) + # gh-124651: test quoted strings + @unittest.skipIf(os.name == 'nt', 'contains invalid characters on Windows') + def test_special_chars_bash(self): + """ + Test that the template strings are quoted properly (bash) + """ + rmtree(self.env_dir) + bash = shutil.which('bash') + if bash is None: + self.skipTest('bash required for this test') + env_name = '"\';&&$e|\'"' + env_dir = os.path.join(os.path.realpath(self.env_dir), env_name) + builder = venv.EnvBuilder(clear=True) + builder.create(env_dir) + activate = os.path.join(env_dir, self.bindir, 'activate') + test_script = os.path.join(self.env_dir, 'test_special_chars.sh') + with open(test_script, "w") as f: + f.write(f'source {shlex.quote(activate)}\n' + 'python -c \'import sys; print(sys.executable)\'\n' + 'python -c \'import os; print(os.environ["VIRTUAL_ENV"])\'\n' + 'deactivate\n') + out, err = check_output([bash, test_script]) + lines = out.splitlines() + self.assertTrue(env_name.encode() in lines[0]) + self.assertEndsWith(lines[1], env_name.encode()) + + # gh-124651: test quoted strings + @unittest.skipIf(os.name == 'nt', 'contains invalid characters on Windows') + def test_special_chars_csh(self): + """ + Test that the template strings are quoted properly (csh) + """ + rmtree(self.env_dir) + csh = shutil.which('tcsh') or shutil.which('csh') + if csh is None: + self.skipTest('csh required for this test') + env_name = '"\';&&$e|\'"' + env_dir = os.path.join(os.path.realpath(self.env_dir), env_name) + builder = venv.EnvBuilder(clear=True) + builder.create(env_dir) + activate = os.path.join(env_dir, self.bindir, 'activate.csh') + test_script = os.path.join(self.env_dir, 'test_special_chars.csh') + with open(test_script, "w") as f: + f.write(f'source {shlex.quote(activate)}\n' + 'python -c \'import sys; print(sys.executable)\'\n' + 'python -c \'import os; print(os.environ["VIRTUAL_ENV"])\'\n' + 'deactivate\n') + out, err = check_output([csh, test_script]) + lines = out.splitlines() + self.assertTrue(env_name.encode() in lines[0]) + self.assertEndsWith(lines[1], env_name.encode()) + + # gh-124651: test quoted strings on Windows + @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows') + def test_special_chars_windows(self): + """ + Test that the template strings are quoted properly on Windows + """ + rmtree(self.env_dir) + env_name = "'&&^$e" + env_dir = os.path.join(os.path.realpath(self.env_dir), env_name) + builder = venv.EnvBuilder(clear=True) + builder.create(env_dir) + activate = os.path.join(env_dir, self.bindir, 'activate.bat') + test_batch = os.path.join(self.env_dir, 'test_special_chars.bat') + with open(test_batch, "w") as f: + f.write('@echo off\n' + f'"{activate}" & ' + f'{self.exe} -c "import sys; print(sys.executable)" & ' + f'{self.exe} -c "import os; print(os.environ[\'VIRTUAL_ENV\'])" & ' + 'deactivate') + out, err = check_output([test_batch]) + lines = out.splitlines() + self.assertTrue(env_name.encode() in lines[0]) + self.assertEndsWith(lines[1], env_name.encode()) + @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows') def test_unicode_in_batch_file(self): """ diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 6bce3081088200..4403f2b1c4ef60 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -11,6 +11,7 @@ import sys import sysconfig import types +import shlex CORE_VENV_DEPS = ('pip', 'setuptools') @@ -394,11 +395,41 @@ def replace_variables(self, text, context): :param context: The information for the environment creation request being processed. """ - text = text.replace('__VENV_DIR__', context.env_dir) - text = text.replace('__VENV_NAME__', context.env_name) - text = text.replace('__VENV_PROMPT__', context.prompt) - text = text.replace('__VENV_BIN_NAME__', context.bin_name) - text = text.replace('__VENV_PYTHON__', context.env_exe) + replacements = { + '__VENV_DIR__': context.env_dir, + '__VENV_NAME__': context.env_name, + '__VENV_PROMPT__': context.prompt, + '__VENV_BIN_NAME__': context.bin_name, + '__VENV_PYTHON__': context.env_exe, + } + + def quote_ps1(s): + """ + This should satisfy PowerShell quoting rules [1], unless the quoted + string is passed directly to Windows native commands [2]. + [1]: https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.co… + [2]: https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.co… + """ + s = s.replace("'", "''") + return f"'{s}'" + + def quote_bat(s): + return s + + # gh-124651: need to quote the template strings properly + quote = shlex.quote + script_path = context.script_path + if script_path.endswith('.ps1'): + quote = quote_ps1 + elif script_path.endswith('.bat'): + quote = quote_bat + else: + # fallbacks to POSIX shell compliant quote + quote = shlex.quote + + replacements = {key: quote(s) for key, s in replacements.items()} + for key, quoted in replacements.items(): + text = text.replace(key, quoted) return text def install_scripts(self, context, path): @@ -438,6 +469,7 @@ def install_scripts(self, context, path): with open(srcfile, 'rb') as f: data = f.read() if not srcfile.endswith(('.exe', '.pdb')): + context.script_path = srcfile try: data = data.decode('utf-8') data = self.replace_variables(data, context) diff --git a/Lib/venv/scripts/common/activate b/Lib/venv/scripts/common/activate index 982da08163b12d..e86442b3578342 100644 --- a/Lib/venv/scripts/common/activate +++ b/Lib/venv/scripts/common/activate @@ -35,11 +35,11 @@ deactivate () { # unset irrelevant variables deactivate nondestructive -VIRTUAL_ENV="__VENV_DIR__" +VIRTUAL_ENV=__VENV_DIR__ export VIRTUAL_ENV _OLD_VIRTUAL_PATH="$PATH" -PATH="$VIRTUAL_ENV/__VENV_BIN_NAME__:$PATH" +PATH="$VIRTUAL_ENV/"__VENV_BIN_NAME__":$PATH" export PATH # unset PYTHONHOME if set @@ -52,9 +52,9 @@ fi if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then _OLD_VIRTUAL_PS1="${PS1:-}" - PS1="__VENV_PROMPT__${PS1:-}" + PS1=__VENV_PROMPT__"${PS1:-}" export PS1 - VIRTUAL_ENV_PROMPT="__VENV_PROMPT__" + VIRTUAL_ENV_PROMPT=__VENV_PROMPT__ export VIRTUAL_ENV_PROMPT fi diff --git a/Lib/venv/scripts/nt/activate.bat b/Lib/venv/scripts/nt/activate.bat index c1c3c82ee37f10..715b21b13fbe35 100644 --- a/Lib/venv/scripts/nt/activate.bat +++ b/Lib/venv/scripts/nt/activate.bat @@ -8,7 +8,7 @@ if defined _OLD_CODEPAGE ( "%SystemRoot%\System32\chcp.com" 65001 > nul ) -set VIRTUAL_ENV=__VENV_DIR__ +set "VIRTUAL_ENV=__VENV_DIR__" if not defined PROMPT set PROMPT=$P$G @@ -24,8 +24,8 @@ set PYTHONHOME= if defined _OLD_VIRTUAL_PATH set PATH=%_OLD_VIRTUAL_PATH% if not defined _OLD_VIRTUAL_PATH set _OLD_VIRTUAL_PATH=%PATH% -set PATH=%VIRTUAL_ENV%\__VENV_BIN_NAME__;%PATH% -set VIRTUAL_ENV_PROMPT=__VENV_PROMPT__ +set "PATH=%VIRTUAL_ENV%\__VENV_BIN_NAME__;%PATH%" +set "VIRTUAL_ENV_PROMPT=__VENV_PROMPT__" :END if defined _OLD_CODEPAGE ( diff --git a/Lib/venv/scripts/posix/activate.csh b/Lib/venv/scripts/posix/activate.csh index d6f697c55ed81c..c47702127eff71 100644 --- a/Lib/venv/scripts/posix/activate.csh +++ b/Lib/venv/scripts/posix/activate.csh @@ -8,17 +8,17 @@ alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PA # Unset irrelevant variables. deactivate nondestructive -setenv VIRTUAL_ENV "__VENV_DIR__" +setenv VIRTUAL_ENV __VENV_DIR__ set _OLD_VIRTUAL_PATH="$PATH" -setenv PATH "$VIRTUAL_ENV/__VENV_BIN_NAME__:$PATH" +setenv PATH "$VIRTUAL_ENV/"__VENV_BIN_NAME__":$PATH" set _OLD_VIRTUAL_PROMPT="$prompt" if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then - set prompt = "__VENV_PROMPT__$prompt" - setenv VIRTUAL_ENV_PROMPT "__VENV_PROMPT__" + set prompt = __VENV_PROMPT__"$prompt" + setenv VIRTUAL_ENV_PROMPT __VENV_PROMPT__ endif alias pydoc python -m pydoc diff --git a/Lib/venv/scripts/posix/activate.fish b/Lib/venv/scripts/posix/activate.fish index 9aa4446005f4d8..dc3a6c88270c18 100644 --- a/Lib/venv/scripts/posix/activate.fish +++ b/Lib/venv/scripts/posix/activate.fish @@ -33,10 +33,10 @@ end # Unset irrelevant variables. deactivate nondestructive -set -gx VIRTUAL_ENV "__VENV_DIR__" +set -gx VIRTUAL_ENV __VENV_DIR__ set -gx _OLD_VIRTUAL_PATH $PATH -set -gx PATH "$VIRTUAL_ENV/__VENV_BIN_NAME__" $PATH +set -gx PATH "$VIRTUAL_ENV/"__VENV_BIN_NAME__ $PATH # Unset PYTHONHOME if set. if set -q PYTHONHOME @@ -56,7 +56,7 @@ if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" set -l old_status $status # Output the venv prompt; color taken from the blue of the Python logo. - printf "%s%s%s" (set_color 4B8BBE) "__VENV_PROMPT__" (set_color normal) + printf "%s%s%s" (set_color 4B8BBE) __VENV_PROMPT__ (set_color normal) # Restore the return status of the previous command. echo "exit $old_status" | . @@ -65,5 +65,5 @@ if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" end set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV" - set -gx VIRTUAL_ENV_PROMPT "__VENV_PROMPT__" + set -gx VIRTUAL_ENV_PROMPT __VENV_PROMPT__ end diff --git a/Misc/NEWS.d/next/Library/2024-09-28-02-03-04.gh-issue-124651.bLBGtH.rst b/Misc/NEWS.d/next/Library/2024-09-28-02-03-04.gh-issue-124651.bLBGtH.rst new file mode 100644 index 00000000000000..17fc9171390dd9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-28-02-03-04.gh-issue-124651.bLBGtH.rst @@ -0,0 +1 @@ +Properly quote template strings in :mod:`venv` activation scripts.

1 0

gh-125783: Add more tests to prevent regressions with the combination of ctypes and metaclasses. (GH-126126)
by encukou Nov. 1, 2024

Nov. 1, 2024

https://github.com/python/cpython/commit/6c67446a6e73ab0e9a26e4360412cbd2f5… commit: 6c67446a6e73ab0e9a26e4360412cbd2f5550e66 branch: main author: Jun Komoda <45822440+junkmd(a)users.noreply.github.com> committer: encukou <encukou(a)gmail.com> date: 2024-11-01T13:50:02+01:00 summary: gh-125783: Add more tests to prevent regressions with the combination of ctypes and metaclasses. (GH-126126) files: M Lib/test/test_ctypes/test_c_simple_type_meta.py diff --git a/Lib/test/test_ctypes/test_c_simple_type_meta.py b/Lib/test/test_ctypes/test_c_simple_type_meta.py index fa5144a3ca01bb..eb77d6d7782478 100644 --- a/Lib/test/test_ctypes/test_c_simple_type_meta.py +++ b/Lib/test/test_ctypes/test_c_simple_type_meta.py @@ -85,3 +85,68 @@ class Sub(CtBase): self.assertIsInstance(POINTER(Sub), p_meta) self.assertTrue(issubclass(POINTER(Sub), Sub)) + + def test_creating_pointer_in_dunder_init_1(self): + class ct_meta(type): + def __init__(self, name, bases, namespace): + super().__init__(name, bases, namespace) + + # Avoid recursion. + # (See test_creating_pointer_in_dunder_new_1) + if bases == (c_void_p,): + return + if issubclass(self, PtrBase): + return + if bases == (object,): + ptr_bases = (self, PtrBase) + else: + ptr_bases = (self, POINTER(bases[0])) + p = p_meta(f"POINTER({self.__name__})", ptr_bases, {}) + ctypes._pointer_type_cache[self] = p + + class p_meta(PyCSimpleType, ct_meta): + pass + + class PtrBase(c_void_p, metaclass=p_meta): + pass + + class CtBase(object, metaclass=ct_meta): + pass + + class Sub(CtBase): + pass + + class Sub2(Sub): + pass + + self.assertIsInstance(POINTER(Sub2), p_meta) + self.assertTrue(issubclass(POINTER(Sub2), Sub2)) + self.assertTrue(issubclass(POINTER(Sub2), POINTER(Sub))) + self.assertTrue(issubclass(POINTER(Sub), POINTER(CtBase))) + + def test_creating_pointer_in_dunder_init_2(self): + class ct_meta(type): + def __init__(self, name, bases, namespace): + super().__init__(name, bases, namespace) + + # Avoid recursion. + # (See test_creating_pointer_in_dunder_new_2) + if isinstance(self, p_meta): + return + p = p_meta(f"POINTER({self.__name__})", (self, c_void_p), {}) + ctypes._pointer_type_cache[self] = p + + class p_meta(PyCSimpleType, ct_meta): + pass + + class Core(object): + pass + + class CtBase(Core, metaclass=ct_meta): + pass + + class Sub(CtBase): + pass + + self.assertIsInstance(POINTER(Sub), p_meta) + self.assertTrue(issubclass(POINTER(Sub), Sub))

1 0

gh-126206: make clinic now forcefully regenerates clinic code (#126244)
by erlend-aasland Nov. 1, 2024

Nov. 1, 2024

https://github.com/python/cpython/commit/43447cb63421fb4db5411c1e74bdd8a4cf… commit: 43447cb63421fb4db5411c1e74bdd8a4cfcf9263 branch: main author: Erlend E. Aasland <erlend(a)python.org> committer: erlend-aasland <erlend.aasland(a)protonmail.com> date: 2024-11-01T08:17:54Z summary: gh-126206: make clinic now forcefully regenerates clinic code (#126244) files: A Misc/NEWS.d/next/Build/2024-10-31-15-37-05.gh-issue-126206.oC6z2i.rst M Makefile.pre.in diff --git a/Makefile.pre.in b/Makefile.pre.in index aa7fa4e29d84c2..b0263f9f4c21da 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -904,7 +904,7 @@ coverage-report: regen-token regen-frozen # Run "Argument Clinic" over all source files .PHONY: clinic clinic: check-clean-src - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/clinic/clinic.py --make --exclude Lib/test/clinic.test.c --srcdir $(srcdir) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/clinic/clinic.py --force --make --exclude Lib/test/clinic.test.c --srcdir $(srcdir) .PHONY: clinic-tests clinic-tests: check-clean-src $(srcdir)/Lib/test/clinic.test.c diff --git a/Misc/NEWS.d/next/Build/2024-10-31-15-37-05.gh-issue-126206.oC6z2i.rst b/Misc/NEWS.d/next/Build/2024-10-31-15-37-05.gh-issue-126206.oC6z2i.rst new file mode 100644 index 00000000000000..24b172e1747403 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-10-31-15-37-05.gh-issue-126206.oC6z2i.rst @@ -0,0 +1,2 @@ +``make clinic`` now runs Argument Clinic using the ``--force`` option, +thus forcefully regenerating generated code.

1 0

[3.12] gh-126259: Fix "unclosed database" warning in sqlite3 doctest (GH-126260) (#126266)
by erlend-aasland Nov. 1, 2024

Nov. 1, 2024

https://github.com/python/cpython/commit/e8ce87219192b9e486198c11b57fea0f57… commit: e8ce87219192b9e486198c11b57fea0f573ea9ea branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: erlend-aasland <erlend.aasland(a)protonmail.com> date: 2024-11-01T07:25:51Z summary: [3.12] gh-126259: Fix "unclosed database" warning in sqlite3 doctest (GH-126260) (#126266) (cherry picked from commit 295262c8ecb085b4fea552bc6229af3551bbaf7a) Co-authored-by: sobolevn <mail(a)sobolevn.me> files: M Doc/library/sqlite3.rst diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index e85a63531a36aa..3078ec29afe1f0 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -2420,6 +2420,7 @@ Some useful URI tricks include: >>> con.execute("CREATE TABLE readonly(data)") Traceback (most recent call last): OperationalError: attempt to write a readonly database + >>> con.close() * Do not implicitly create a new database file if it does not already exist; will raise :exc:`~sqlite3.OperationalError` if unable to create a new file:

1 0

[3.13] gh-126259: Fix "unclosed database" warning in sqlite3 doctest (GH-126260) (#126265)
by erlend-aasland Nov. 1, 2024

Nov. 1, 2024

https://github.com/python/cpython/commit/2e20a49cae69a11c69baf2e3d405918c98… commit: 2e20a49cae69a11c69baf2e3d405918c98bf8bdb branch: 3.13 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: erlend-aasland <erlend.aasland(a)protonmail.com> date: 2024-11-01T07:23:11Z summary: [3.13] gh-126259: Fix "unclosed database" warning in sqlite3 doctest (GH-126260) (#126265) (cherry picked from commit 295262c8ecb085b4fea552bc6229af3551bbaf7a) Co-authored-by: sobolevn <mail(a)sobolevn.me> files: M Doc/library/sqlite3.rst diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 91250c45410597..0d2004497882e2 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -2460,6 +2460,7 @@ Some useful URI tricks include: >>> con.execute("CREATE TABLE readonly(data)") Traceback (most recent call last): OperationalError: attempt to write a readonly database + >>> con.close() * Do not implicitly create a new database file if it does not already exist; will raise :exc:`~sqlite3.OperationalError` if unable to create a new file:

1 0

gh-126259: Fix "unclosed database" warning in sqlite3 doctest (#126260)
by erlend-aasland Nov. 1, 2024

Nov. 1, 2024

https://github.com/python/cpython/commit/295262c8ecb085b4fea552bc6229af3551… commit: 295262c8ecb085b4fea552bc6229af3551bbaf7a branch: main author: sobolevn <mail(a)sobolevn.me> committer: erlend-aasland <erlend.aasland(a)protonmail.com> date: 2024-11-01T08:17:05+01:00 summary: gh-126259: Fix "unclosed database" warning in sqlite3 doctest (#126260) files: M Doc/library/sqlite3.rst diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index fc0383823a172b..096892b605b99c 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -2442,6 +2442,7 @@ Some useful URI tricks include: >>> con.execute("CREATE TABLE readonly(data)") Traceback (most recent call last): OperationalError: attempt to write a readonly database + >>> con.close() * Do not implicitly create a new database file if it does not already exist; will raise :exc:`~sqlite3.OperationalError` if unable to create a new file:

1 0

GH-125413: Add `pathlib.Path.scandir()` method (#126060)
by barneygale Nov. 1, 2024

Nov. 1, 2024

https://github.com/python/cpython/commit/260843df1bd8a28596b9a377d8266e2547… commit: 260843df1bd8a28596b9a377d8266e2547f7eedc branch: main author: Barney Gale <barney.gale(a)gmail.com> committer: barneygale <barney.gale(a)gmail.com> date: 2024-11-01T01:19:01Z summary: GH-125413: Add `pathlib.Path.scandir()` method (#126060) Add `pathlib.Path.scandir()` as a trivial wrapper of `os.scandir()`. This will be used to implement several `PathBase` methods more efficiently, including methods that provide `Path.copy()`. files: A Misc/NEWS.d/next/Library/2024-10-28-01-24-52.gh-issue-125413.Jat5kq.rst M Doc/library/pathlib.rst M Doc/whatsnew/3.14.rst M Lib/pathlib/_abc.py M Lib/pathlib/_local.py M Lib/test/test_pathlib/test_pathlib_abc.py diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 4380122eb1be7d..b6fb36554f7cec 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -1289,6 +1289,35 @@ Reading directories raised. +.. method:: Path.scandir() + + When the path points to a directory, return an iterator of + :class:`os.DirEntry` objects corresponding to entries in the directory. The + returned iterator supports the :term:`context manager` protocol. It is + implemented using :func:`os.scandir` and gives the same guarantees. + + Using :meth:`~Path.scandir` instead of :meth:`~Path.iterdir` can + significantly increase the performance of code that also needs file type or + file attribute information, because :class:`os.DirEntry` objects expose + this information if the operating system provides it when scanning a + directory. + + The following example displays the names of subdirectories. The + ``entry.is_dir()`` check will generally not make an additional system call:: + + >>> p = Path('docs') + >>> with p.scandir() as entries: + ... for entry in entries: + ... if entry.is_dir(): + ... entry.name + ... + '_templates' + '_build' + '_static' + + .. versionadded:: 3.14 + + .. method:: Path.glob(pattern, *, case_sensitive=None, recurse_symlinks=False) Glob the given relative *pattern* in the directory represented by this path, diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 7f9e3107a6e1a0..48314f9c98c036 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -380,6 +380,12 @@ pathlib (Contributed by Barney Gale in :gh:`73991`.) +* Add :meth:`pathlib.Path.scandir` to scan a directory and return an iterator + of :class:`os.DirEntry` objects. This is exactly equivalent to calling + :func:`os.scandir` on a path object. + + (Contributed by Barney Gale in :gh:`125413`.) + pdb --- diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 11c8018b28f26b..dfff8b460d1bf1 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -639,13 +639,23 @@ def write_text(self, data, encoding=None, errors=None, newline=None): with self.open(mode='w', encoding=encoding, errors=errors, newline=newline) as f: return f.write(data) + def scandir(self): + """Yield os.DirEntry objects of the directory contents. + + The children are yielded in arbitrary order, and the + special entries '.' and '..' are not included. + """ + raise UnsupportedOperation(self._unsupported_msg('scandir()')) + def iterdir(self): """Yield path objects of the directory contents. The children are yielded in arbitrary order, and the special entries '.' and '..' are not included. """ - raise UnsupportedOperation(self._unsupported_msg('iterdir()')) + with self.scandir() as entries: + names = [entry.name for entry in entries] + return map(self.joinpath, names) def _glob_selector(self, parts, case_sensitive, recurse_symlinks): if case_sensitive is None: diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index a78997179820b1..ef072b83d96904 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -615,6 +615,14 @@ def _filter_trailing_slash(self, paths): path_str = path_str[:-1] yield path_str + def scandir(self): + """Yield os.DirEntry objects of the directory contents. + + The children are yielded in arbitrary order, and the + special entries '.' and '..' are not included. + """ + return os.scandir(self) + def iterdir(self): """Yield path objects of the directory contents. diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index 08355a71453807..11e34f5d378a58 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -1,4 +1,5 @@ import collections +import contextlib import io import os import errno @@ -1424,6 +1425,24 @@ def close(self): 'st_mode st_ino st_dev st_nlink st_uid st_gid st_size st_atime st_mtime st_ctime') +class DummyDirEntry: + """ + Minimal os.DirEntry-like object. Returned from DummyPath.scandir(). + """ + __slots__ = ('name', '_is_symlink', '_is_dir') + + def __init__(self, name, is_symlink, is_dir): + self.name = name + self._is_symlink = is_symlink + self._is_dir = is_dir + + def is_symlink(self): + return self._is_symlink + + def is_dir(self, *, follow_symlinks=True): + return self._is_dir and (follow_symlinks or not self._is_symlink) + + class DummyPath(PathBase): """ Simple implementation of PathBase that keeps files and directories in @@ -1491,14 +1510,25 @@ def open(self, mode='r', buffering=-1, encoding=None, stream = io.TextIOWrapper(stream, encoding=encoding, errors=errors, newline=newline) return stream - def iterdir(self): - path = str(self.resolve()) - if path in self._files: - raise NotADirectoryError(errno.ENOTDIR, "Not a directory", path) - elif path in self._directories: - return iter([self / name for name in self._directories[path]]) + @contextlib.contextmanager + def scandir(self): + path = self.resolve() + path_str = str(path) + if path_str in self._files: + raise NotADirectoryError(errno.ENOTDIR, "Not a directory", path_str) + elif path_str in self._directories: + yield iter([path.joinpath(name)._dir_entry for name in self._directories[path_str]]) else: - raise FileNotFoundError(errno.ENOENT, "File not found", path) + raise FileNotFoundError(errno.ENOENT, "File not found", path_str) + + @property + def _dir_entry(self): + path_str = str(self) + is_symlink = path_str in self._symlinks + is_directory = (path_str in self._directories + if not is_symlink + else self._symlinks[path_str][1]) + return DummyDirEntry(self.name, is_symlink, is_directory) def mkdir(self, mode=0o777, parents=False, exist_ok=False): path = str(self.parent.resolve() / self.name) @@ -1602,7 +1632,7 @@ def setUp(self): if self.can_symlink: p.joinpath('linkA').symlink_to('fileA') p.joinpath('brokenLink').symlink_to('non-existing') - p.joinpath('linkB').symlink_to('dirB') + p.joinpath('linkB').symlink_to('dirB', target_is_directory=True) p.joinpath('dirA', 'linkC').symlink_to(parser.join('..', 'dirB')) p.joinpath('dirB', 'linkD').symlink_to(parser.join('..', 'dirB')) p.joinpath('brokenLinkLoop').symlink_to('brokenLinkLoop') @@ -2187,6 +2217,23 @@ def test_iterdir_nodir(self): self.assertIn(cm.exception.errno, (errno.ENOTDIR, errno.ENOENT, errno.EINVAL)) + def test_scandir(self): + p = self.cls(self.base) + with p.scandir() as entries: + self.assertTrue(list(entries)) + with p.scandir() as entries: + for entry in entries: + child = p / entry.name + self.assertIsNotNone(entry) + self.assertEqual(entry.name, child.name) + self.assertEqual(entry.is_symlink(), + child.is_symlink()) + self.assertEqual(entry.is_dir(follow_symlinks=False), + child.is_dir(follow_symlinks=False)) + if entry.name != 'brokenLinkLoop': + self.assertEqual(entry.is_dir(), child.is_dir()) + + def test_glob_common(self): def _check(glob, expected): self.assertEqual(set(glob), { P(self.base, q) for q in expected }) @@ -3038,7 +3085,7 @@ class DummyPathWithSymlinks(DummyPath): def readlink(self): path = str(self.parent.resolve() / self.name) if path in self._symlinks: - return self.with_segments(self._symlinks[path]) + return self.with_segments(self._symlinks[path][0]) elif path in self._files or path in self._directories: raise OSError(errno.EINVAL, "Not a symlink", path) else: @@ -3050,7 +3097,7 @@ def symlink_to(self, target, target_is_directory=False): if path in self._symlinks: raise FileExistsError(errno.EEXIST, "File exists", path) self._directories[parent].add(self.name) - self._symlinks[path] = str(target) + self._symlinks[path] = str(target), target_is_directory class DummyPathWithSymlinksTest(DummyPathTest): diff --git a/Misc/NEWS.d/next/Library/2024-10-28-01-24-52.gh-issue-125413.Jat5kq.rst b/Misc/NEWS.d/next/Library/2024-10-28-01-24-52.gh-issue-125413.Jat5kq.rst new file mode 100644 index 00000000000000..ddf1f9725d9695 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-28-01-24-52.gh-issue-125413.Jat5kq.rst @@ -0,0 +1,3 @@ +Add :meth:`pathlib.Path.scandir` method to efficiently fetch directory +children and their file attributes. This is a trivial wrapper of +:func:`os.scandir`.

1 0