Mailman 3 March 2025 - Python-checkins

GH-127381: pathlib ABCs: remove `ReadablePath.exists()` and `is_*()` (#130520)
by barneygale March 1, 2025

March 1, 2025

https://github.com/python/cpython/commit/a55dffd66dbddfd50c8f3de195218d041d… commit: a55dffd66dbddfd50c8f3de195218d041d26bd3c branch: main author: Barney Gale <barney.gale(a)gmail.com> committer: barneygale <barney.gale(a)gmail.com> date: 2025-03-01T21:24:19Z summary: GH-127381: pathlib ABCs: remove `ReadablePath.exists()` and `is_*()` (#130520) Remove `ReadablePath` methods duplicated by `ReadablePath.info`. To be specific, we remove `exists()`, `is_dir()`, `is_file()` and `is_symlink()`. The public `Path` class retains these methods. files: M Lib/pathlib/_abc.py M Lib/test/test_pathlib/test_pathlib.py M Lib/test/test_pathlib/test_pathlib_abc.py diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 930701d4789f5c..cb5af3230bfd67 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -219,38 +219,6 @@ def info(self): """ raise NotImplementedError - def exists(self, *, follow_symlinks=True): - """ - Whether this path exists. - - This method normally follows symlinks; to check whether a symlink exists, - add the argument follow_symlinks=False. - """ - info = self.joinpath().info - return info.exists(follow_symlinks=follow_symlinks) - - def is_dir(self, *, follow_symlinks=True): - """ - Whether this path is a directory. - """ - info = self.joinpath().info - return info.is_dir(follow_symlinks=follow_symlinks) - - def is_file(self, *, follow_symlinks=True): - """ - Whether this path is a regular file (also True for symlinks pointing - to regular files). - """ - info = self.joinpath().info - return info.is_file(follow_symlinks=follow_symlinks) - - def is_symlink(self): - """ - Whether this path is a symbolic link. - """ - info = self.joinpath().info - return info.is_symlink() - @abstractmethod def __open_rb__(self, buffering=-1): """ diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index 182d054de9d234..4bb62daecb6356 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -2453,6 +2453,93 @@ def test_lstat_nosymlink(self): st = p.stat() self.assertEqual(st, p.lstat()) + def test_exists(self): + P = self.cls + p = P(self.base) + self.assertIs(True, p.exists()) + self.assertIs(True, (p / 'dirA').exists()) + self.assertIs(True, (p / 'fileA').exists()) + self.assertIs(False, (p / 'fileA' / 'bah').exists()) + if self.can_symlink: + self.assertIs(True, (p / 'linkA').exists()) + self.assertIs(True, (p / 'linkB').exists()) + self.assertIs(True, (p / 'linkB' / 'fileB').exists()) + self.assertIs(False, (p / 'linkA' / 'bah').exists()) + self.assertIs(False, (p / 'brokenLink').exists()) + self.assertIs(True, (p / 'brokenLink').exists(follow_symlinks=False)) + self.assertIs(False, (p / 'foo').exists()) + self.assertIs(False, P('/xyzzy').exists()) + self.assertIs(False, P(self.base + '\udfff').exists()) + self.assertIs(False, P(self.base + '\x00').exists()) + + def test_is_dir(self): + P = self.cls(self.base) + self.assertTrue((P / 'dirA').is_dir()) + self.assertFalse((P / 'fileA').is_dir()) + self.assertFalse((P / 'non-existing').is_dir()) + self.assertFalse((P / 'fileA' / 'bah').is_dir()) + if self.can_symlink: + self.assertFalse((P / 'linkA').is_dir()) + self.assertTrue((P / 'linkB').is_dir()) + self.assertFalse((P/ 'brokenLink').is_dir()) + self.assertFalse((P / 'dirA\udfff').is_dir()) + self.assertFalse((P / 'dirA\x00').is_dir()) + + def test_is_dir_no_follow_symlinks(self): + P = self.cls(self.base) + self.assertTrue((P / 'dirA').is_dir(follow_symlinks=False)) + self.assertFalse((P / 'fileA').is_dir(follow_symlinks=False)) + self.assertFalse((P / 'non-existing').is_dir(follow_symlinks=False)) + self.assertFalse((P / 'fileA' / 'bah').is_dir(follow_symlinks=False)) + if self.can_symlink: + self.assertFalse((P / 'linkA').is_dir(follow_symlinks=False)) + self.assertFalse((P / 'linkB').is_dir(follow_symlinks=False)) + self.assertFalse((P/ 'brokenLink').is_dir(follow_symlinks=False)) + self.assertFalse((P / 'dirA\udfff').is_dir(follow_symlinks=False)) + self.assertFalse((P / 'dirA\x00').is_dir(follow_symlinks=False)) + + def test_is_file(self): + P = self.cls(self.base) + self.assertTrue((P / 'fileA').is_file()) + self.assertFalse((P / 'dirA').is_file()) + self.assertFalse((P / 'non-existing').is_file()) + self.assertFalse((P / 'fileA' / 'bah').is_file()) + if self.can_symlink: + self.assertTrue((P / 'linkA').is_file()) + self.assertFalse((P / 'linkB').is_file()) + self.assertFalse((P/ 'brokenLink').is_file()) + self.assertFalse((P / 'fileA\udfff').is_file()) + self.assertFalse((P / 'fileA\x00').is_file()) + + def test_is_file_no_follow_symlinks(self): + P = self.cls(self.base) + self.assertTrue((P / 'fileA').is_file(follow_symlinks=False)) + self.assertFalse((P / 'dirA').is_file(follow_symlinks=False)) + self.assertFalse((P / 'non-existing').is_file(follow_symlinks=False)) + self.assertFalse((P / 'fileA' / 'bah').is_file(follow_symlinks=False)) + if self.can_symlink: + self.assertFalse((P / 'linkA').is_file(follow_symlinks=False)) + self.assertFalse((P / 'linkB').is_file(follow_symlinks=False)) + self.assertFalse((P/ 'brokenLink').is_file(follow_symlinks=False)) + self.assertFalse((P / 'fileA\udfff').is_file(follow_symlinks=False)) + self.assertFalse((P / 'fileA\x00').is_file(follow_symlinks=False)) + + def test_is_symlink(self): + P = self.cls(self.base) + self.assertFalse((P / 'fileA').is_symlink()) + self.assertFalse((P / 'dirA').is_symlink()) + self.assertFalse((P / 'non-existing').is_symlink()) + self.assertFalse((P / 'fileA' / 'bah').is_symlink()) + if self.can_symlink: + self.assertTrue((P / 'linkA').is_symlink()) + self.assertTrue((P / 'linkB').is_symlink()) + self.assertTrue((P/ 'brokenLink').is_symlink()) + self.assertIs((P / 'fileA\udfff').is_file(), False) + self.assertIs((P / 'fileA\x00').is_file(), False) + if self.can_symlink: + self.assertIs((P / 'linkA\udfff').is_file(), False) + self.assertIs((P / 'linkA\x00').is_file(), False) + def test_is_junction_false(self): P = self.cls(self.base) self.assertFalse((P / 'fileA').is_junction()) diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index b64924d06d420b..efc6c17962e261 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -1007,25 +1007,6 @@ def test_is_readable(self): p = self.cls(self.base) self.assertIsInstance(p, ReadablePath) - def test_exists(self): - P = self.cls - p = P(self.base) - self.assertIs(True, p.exists()) - self.assertIs(True, (p / 'dirA').exists()) - self.assertIs(True, (p / 'fileA').exists()) - self.assertIs(False, (p / 'fileA' / 'bah').exists()) - if self.can_symlink: - self.assertIs(True, (p / 'linkA').exists()) - self.assertIs(True, (p / 'linkB').exists()) - self.assertIs(True, (p / 'linkB' / 'fileB').exists()) - self.assertIs(False, (p / 'linkA' / 'bah').exists()) - self.assertIs(False, (p / 'brokenLink').exists()) - self.assertIs(True, (p / 'brokenLink').exists(follow_symlinks=False)) - self.assertIs(False, (p / 'foo').exists()) - self.assertIs(False, P('/xyzzy').exists()) - self.assertIs(False, P(self.base + '\udfff').exists()) - self.assertIs(False, P(self.base + '\x00').exists()) - def test_magic_open(self): p = self.cls(self.base) with magic_open(p / 'fileA', 'r') as f: @@ -1058,17 +1039,8 @@ def test_iterdir_nodir(self): def test_iterdir_info(self): p = self.cls(self.base) for child in p.iterdir(): - info = child.info - self.assertIsInstance(info, PathInfo) - self.assertEqual(info.exists(), child.exists()) - self.assertEqual(info.is_dir(), child.is_dir()) - self.assertEqual(info.is_file(), child.is_file()) - self.assertEqual(info.is_symlink(), child.is_symlink()) - self.assertTrue(info.exists(follow_symlinks=False)) - self.assertEqual(info.is_dir(follow_symlinks=False), - child.is_dir(follow_symlinks=False)) - self.assertEqual(info.is_file(follow_symlinks=False), - child.is_file(follow_symlinks=False)) + self.assertIsInstance(child.info, PathInfo) + self.assertTrue(child.info.exists(follow_symlinks=False)) def test_glob_common(self): def _check(glob, expected): @@ -1248,74 +1220,6 @@ def test_info_is_symlink(self): self.assertFalse((p / 'fileA\udfff').info.is_symlink()) self.assertFalse((p / 'fileA\x00').info.is_symlink()) - def test_is_dir(self): - P = self.cls(self.base) - self.assertTrue((P / 'dirA').is_dir()) - self.assertFalse((P / 'fileA').is_dir()) - self.assertFalse((P / 'non-existing').is_dir()) - self.assertFalse((P / 'fileA' / 'bah').is_dir()) - if self.can_symlink: - self.assertFalse((P / 'linkA').is_dir()) - self.assertTrue((P / 'linkB').is_dir()) - self.assertFalse((P/ 'brokenLink').is_dir()) - self.assertFalse((P / 'dirA\udfff').is_dir()) - self.assertFalse((P / 'dirA\x00').is_dir()) - - def test_is_dir_no_follow_symlinks(self): - P = self.cls(self.base) - self.assertTrue((P / 'dirA').is_dir(follow_symlinks=False)) - self.assertFalse((P / 'fileA').is_dir(follow_symlinks=False)) - self.assertFalse((P / 'non-existing').is_dir(follow_symlinks=False)) - self.assertFalse((P / 'fileA' / 'bah').is_dir(follow_symlinks=False)) - if self.can_symlink: - self.assertFalse((P / 'linkA').is_dir(follow_symlinks=False)) - self.assertFalse((P / 'linkB').is_dir(follow_symlinks=False)) - self.assertFalse((P/ 'brokenLink').is_dir(follow_symlinks=False)) - self.assertFalse((P / 'dirA\udfff').is_dir(follow_symlinks=False)) - self.assertFalse((P / 'dirA\x00').is_dir(follow_symlinks=False)) - - def test_is_file(self): - P = self.cls(self.base) - self.assertTrue((P / 'fileA').is_file()) - self.assertFalse((P / 'dirA').is_file()) - self.assertFalse((P / 'non-existing').is_file()) - self.assertFalse((P / 'fileA' / 'bah').is_file()) - if self.can_symlink: - self.assertTrue((P / 'linkA').is_file()) - self.assertFalse((P / 'linkB').is_file()) - self.assertFalse((P/ 'brokenLink').is_file()) - self.assertFalse((P / 'fileA\udfff').is_file()) - self.assertFalse((P / 'fileA\x00').is_file()) - - def test_is_file_no_follow_symlinks(self): - P = self.cls(self.base) - self.assertTrue((P / 'fileA').is_file(follow_symlinks=False)) - self.assertFalse((P / 'dirA').is_file(follow_symlinks=False)) - self.assertFalse((P / 'non-existing').is_file(follow_symlinks=False)) - self.assertFalse((P / 'fileA' / 'bah').is_file(follow_symlinks=False)) - if self.can_symlink: - self.assertFalse((P / 'linkA').is_file(follow_symlinks=False)) - self.assertFalse((P / 'linkB').is_file(follow_symlinks=False)) - self.assertFalse((P/ 'brokenLink').is_file(follow_symlinks=False)) - self.assertFalse((P / 'fileA\udfff').is_file(follow_symlinks=False)) - self.assertFalse((P / 'fileA\x00').is_file(follow_symlinks=False)) - - def test_is_symlink(self): - P = self.cls(self.base) - self.assertFalse((P / 'fileA').is_symlink()) - self.assertFalse((P / 'dirA').is_symlink()) - self.assertFalse((P / 'non-existing').is_symlink()) - self.assertFalse((P / 'fileA' / 'bah').is_symlink()) - if self.can_symlink: - self.assertTrue((P / 'linkA').is_symlink()) - self.assertTrue((P / 'linkB').is_symlink()) - self.assertTrue((P/ 'brokenLink').is_symlink()) - self.assertIs((P / 'fileA\udfff').is_file(), False) - self.assertIs((P / 'fileA\x00').is_file(), False) - if self.can_symlink: - self.assertIs((P / 'linkA\udfff').is_file(), False) - self.assertIs((P / 'linkA\x00').is_file(), False) - class WritablePathTest(JoinablePathTest): cls = DummyWritablePath

1 0

[3.12] gh-130160: use `.. program::` directive for documenting `cProfile` CLI (GH-130314) (#130746)
by sobolevn March 1, 2025

March 1, 2025

https://github.com/python/cpython/commit/c19294436242dff8efa912425af9adcda9… commit: c19294436242dff8efa912425af9adcda97b1015 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: sobolevn <mail(a)sobolevn.me> date: 2025-03-01T20:46:45Z summary: [3.12] gh-130160: use `.. program::` directive for documenting `cProfile` CLI (GH-130314) (#130746) gh-130160: use `.. program::` directive for documenting `cProfile` CLI (GH-130314) (cherry picked from commit 5181ddb29f969c1718f3f4f9618c260807b2886c) Co-authored-by: Apostol Fet <90645107+ApostolFet(a)users.noreply.github.com> Co-authored-by: Bénédikt Tran <10796600+picnixz(a)users.noreply.github.com> Co-authored-by: Adam Turner <9087854+AA-Turner(a)users.noreply.github.com> files: M Doc/library/profile.rst diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index b89655ea472dee..a2887c59283227 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -123,23 +123,32 @@ them in various ways. .. _profile-cli: +.. program:: cProfile + The files :mod:`cProfile` and :mod:`profile` can also be invoked as a script to profile another script. For example:: python -m cProfile [-o output_file] [-s sort_order] (-m module | myscript.py) -``-o`` writes the profile results to a file instead of to stdout +.. option:: -o <output_file> + + Writes the profile results to a file instead of to stdout. + +.. option:: -s <sort_order> + + Specifies one of the :func:`~pstats.Stats.sort_stats` sort values + to sort the output by. + This only applies when :option:`-o <cProfile -o>` is not supplied. -``-s`` specifies one of the :func:`~pstats.Stats.sort_stats` sort values to sort -the output by. This only applies when ``-o`` is not supplied. +.. option:: -m <module> -``-m`` specifies that a module is being profiled instead of a script. + Specifies that a module is being profiled instead of a script. -.. versionadded:: 3.7 - Added the ``-m`` option to :mod:`cProfile`. + .. versionadded:: 3.7 + Added the ``-m`` option to :mod:`cProfile`. -.. versionadded:: 3.8 - Added the ``-m`` option to :mod:`profile`. + .. versionadded:: 3.8 + Added the ``-m`` option to :mod:`profile`. The :mod:`pstats` module's :class:`~pstats.Stats` class has a variety of methods for manipulating and printing the data saved into a profile results file::

1 0

[3.13] gh-130160: use `.. program::` directive for documenting `cProfile` CLI (GH-130314) (#130745)
by sobolevn March 1, 2025

March 1, 2025

https://github.com/python/cpython/commit/8edf17f260121357f323359d19cda8dd77… commit: 8edf17f260121357f323359d19cda8dd77bf5558 branch: 3.13 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: sobolevn <mail(a)sobolevn.me> date: 2025-03-01T20:45:33Z summary: [3.13] gh-130160: use `.. program::` directive for documenting `cProfile` CLI (GH-130314) (#130745) gh-130160: use `.. program::` directive for documenting `cProfile` CLI (GH-130314) (cherry picked from commit 5181ddb29f969c1718f3f4f9618c260807b2886c) Co-authored-by: Apostol Fet <90645107+ApostolFet(a)users.noreply.github.com> Co-authored-by: Bénédikt Tran <10796600+picnixz(a)users.noreply.github.com> Co-authored-by: Adam Turner <9087854+AA-Turner(a)users.noreply.github.com> files: M Doc/library/profile.rst diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index 3334833eba6b8c..b6e51dffc40157 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -123,23 +123,32 @@ them in various ways. .. _profile-cli: +.. program:: cProfile + The files :mod:`cProfile` and :mod:`profile` can also be invoked as a script to profile another script. For example:: python -m cProfile [-o output_file] [-s sort_order] (-m module | myscript.py) -``-o`` writes the profile results to a file instead of to stdout +.. option:: -o <output_file> + + Writes the profile results to a file instead of to stdout. + +.. option:: -s <sort_order> + + Specifies one of the :func:`~pstats.Stats.sort_stats` sort values + to sort the output by. + This only applies when :option:`-o <cProfile -o>` is not supplied. -``-s`` specifies one of the :func:`~pstats.Stats.sort_stats` sort values to sort -the output by. This only applies when ``-o`` is not supplied. +.. option:: -m <module> -``-m`` specifies that a module is being profiled instead of a script. + Specifies that a module is being profiled instead of a script. -.. versionadded:: 3.7 - Added the ``-m`` option to :mod:`cProfile`. + .. versionadded:: 3.7 + Added the ``-m`` option to :mod:`cProfile`. -.. versionadded:: 3.8 - Added the ``-m`` option to :mod:`profile`. + .. versionadded:: 3.8 + Added the ``-m`` option to :mod:`profile`. The :mod:`pstats` module's :class:`~pstats.Stats` class has a variety of methods for manipulating and printing the data saved into a profile results file::

1 0

gh-130160: use `.. program::` directive for documenting `cProfile` CLI (#130314)
by sobolevn March 1, 2025

March 1, 2025

https://github.com/python/cpython/commit/5181ddb29f969c1718f3f4f9618c260807… commit: 5181ddb29f969c1718f3f4f9618c260807b2886c branch: main author: Apostol Fet <90645107+ApostolFet(a)users.noreply.github.com> committer: sobolevn <mail(a)sobolevn.me> date: 2025-03-01T23:39:25+03:00 summary: gh-130160: use `.. program::` directive for documenting `cProfile` CLI (#130314) Co-authored-by: Bénédikt Tran <10796600+picnixz(a)users.noreply.github.com> Co-authored-by: Adam Turner <9087854+AA-Turner(a)users.noreply.github.com> files: M Doc/library/profile.rst diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index 3334833eba6b8c..b6e51dffc40157 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -123,23 +123,32 @@ them in various ways. .. _profile-cli: +.. program:: cProfile + The files :mod:`cProfile` and :mod:`profile` can also be invoked as a script to profile another script. For example:: python -m cProfile [-o output_file] [-s sort_order] (-m module | myscript.py) -``-o`` writes the profile results to a file instead of to stdout +.. option:: -o <output_file> + + Writes the profile results to a file instead of to stdout. + +.. option:: -s <sort_order> + + Specifies one of the :func:`~pstats.Stats.sort_stats` sort values + to sort the output by. + This only applies when :option:`-o <cProfile -o>` is not supplied. -``-s`` specifies one of the :func:`~pstats.Stats.sort_stats` sort values to sort -the output by. This only applies when ``-o`` is not supplied. +.. option:: -m <module> -``-m`` specifies that a module is being profiled instead of a script. + Specifies that a module is being profiled instead of a script. -.. versionadded:: 3.7 - Added the ``-m`` option to :mod:`cProfile`. + .. versionadded:: 3.7 + Added the ``-m`` option to :mod:`cProfile`. -.. versionadded:: 3.8 - Added the ``-m`` option to :mod:`profile`. + .. versionadded:: 3.8 + Added the ``-m`` option to :mod:`profile`. The :mod:`pstats` module's :class:`~pstats.Stats` class has a variety of methods for manipulating and printing the data saved into a profile results file::

1 0

gh-129401: Test repr rlock failing randomly (#129959)
by colesbury March 1, 2025

March 1, 2025

https://github.com/python/cpython/commit/a98a6bd1128663fbe58c0c73d468710245… commit: a98a6bd1128663fbe58c0c73d468710245a57ad6 branch: main author: Duprat <yduprat(a)gmail.com> committer: colesbury <colesbury(a)gmail.com> date: 2025-03-01T15:28:14-05:00 summary: gh-129401: Test repr rlock failing randomly (#129959) Fix and simplify a test of `test_repr_rlock` about multiprocessing.RLock primitive. files: A Misc/NEWS.d/next/Tests/2025-02-10-14-34-29.gh-issue-129401.Cq6Ruy.rst M Lib/test/_test_multiprocessing.py diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 5dd89bd5af7daa..cafbc757904f64 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -1521,14 +1521,11 @@ def test_repr_rlock(self): for i in range(n): self.assertIn(f'<RLock(MainProcess|T{i+1}, {i+1})>', l) - - t = threading.Thread(target=self._acquire_release, - args=(lock, 0.2), - name=f'T1') + rlock = self.RLock() + t = threading.Thread(target=rlock.acquire) t.start() - time.sleep(0.1) - self.assertEqual('<RLock(SomeOtherThread, nonzero)>', repr(lock)) - time.sleep(0.2) + t.join() + self.assertEqual('<RLock(SomeOtherThread, nonzero)>', repr(rlock)) pname = 'P1' l = multiprocessing.Manager().list() @@ -1539,14 +1536,11 @@ def test_repr_rlock(self): p.join() self.assertEqual(f'<RLock({pname}, 1)>', l[0]) - event = self.Event() - lock = self.RLock() - p = self.Process(target=self._acquire_event, - args=(lock, event)) + rlock = self.RLock() + p = self.Process(target=self._acquire, args=(rlock,)) p.start() - event.wait() - self.assertEqual('<RLock(SomeOtherProcess, nonzero)>', repr(lock)) p.join() + self.assertEqual('<RLock(SomeOtherProcess, nonzero)>', repr(rlock)) def test_rlock(self): lock = self.RLock() diff --git a/Misc/NEWS.d/next/Tests/2025-02-10-14-34-29.gh-issue-129401.Cq6Ruy.rst b/Misc/NEWS.d/next/Tests/2025-02-10-14-34-29.gh-issue-129401.Cq6Ruy.rst new file mode 100644 index 00000000000000..7b87d5455c1a70 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2025-02-10-14-34-29.gh-issue-129401.Cq6Ruy.rst @@ -0,0 +1 @@ +Fix a flaky test in ``test_repr_rlock`` that checks the representation of :class:`multiprocessing.RLock`.

1 0

Revert "GH-116380: Speed up `glob.[i]glob()` by making fewer system calls. (#116392)" (#130743)
by barneygale March 1, 2025

March 1, 2025

https://github.com/python/cpython/commit/5326c27fc674713879747e61a8deca7a66… commit: 5326c27fc674713879747e61a8deca7a66e69754 branch: main author: Barney Gale <barney.gale(a)gmail.com> committer: barneygale <barney.gale(a)gmail.com> date: 2025-03-01T20:04:01Z summary: Revert "GH-116380: Speed up `glob.[i]glob()` by making fewer system calls. (#116392)" (#130743) This broke tests on the 'aarch64 Fedora Stable Clang Installed 3.x' and 'AMD64 Fedora Stable Clang Installed 3.x' build bots. This reverts commit da4899b94a9a9083fed4972b2473546e0d997727. files: D Misc/NEWS.d/next/Library/2024-03-05-23-08-11.gh-issue-116380.56HU7I.rst M Doc/library/glob.rst M Doc/whatsnew/3.14.rst M Lib/glob.py M Lib/pathlib/_abc.py M Lib/pathlib/_local.py M Lib/test/test_glob.py diff --git a/Doc/library/glob.rst b/Doc/library/glob.rst index 849316553e408a..684466d354aef8 100644 --- a/Doc/library/glob.rst +++ b/Doc/library/glob.rst @@ -75,6 +75,10 @@ The :mod:`glob` module defines the following functions: Using the "``**``" pattern in large directory trees may consume an inordinate amount of time. + .. note:: + This function may return duplicate path names if *pathname* + contains multiple "``**``" patterns and *recursive* is true. + .. versionchanged:: 3.5 Support for recursive globs using "``**``". @@ -84,11 +88,6 @@ The :mod:`glob` module defines the following functions: .. versionchanged:: 3.11 Added the *include_hidden* parameter. - .. versionchanged:: 3.14 - Matching path names are returned only once. In previous versions, this - function may return duplicate path names if *pathname* contains multiple - "``**``" patterns and *recursive* is true. - .. function:: iglob(pathname, *, root_dir=None, dir_fd=None, recursive=False, \ include_hidden=False) @@ -99,6 +98,10 @@ The :mod:`glob` module defines the following functions: .. audit-event:: glob.glob pathname,recursive glob.iglob .. audit-event:: glob.glob/2 pathname,recursive,root_dir,dir_fd glob.iglob + .. note:: + This function may return duplicate path names if *pathname* + contains multiple "``**``" patterns and *recursive* is true. + .. versionchanged:: 3.5 Support for recursive globs using "``**``". @@ -108,11 +111,6 @@ The :mod:`glob` module defines the following functions: .. versionchanged:: 3.11 Added the *include_hidden* parameter. - .. versionchanged:: 3.14 - Matching path names are yielded only once. In previous versions, this - function may yield duplicate path names if *pathname* contains multiple - "``**``" patterns and *recursive* is true. - .. function:: escape(pathname) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 3c876a193fad32..dbd59a9d7be150 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -968,14 +968,6 @@ base64 (Contributed by Bénédikt Tran, Chris Markiewicz, and Adam Turner in :gh:`118761`.) -glob ----- - -* Reduce the number of system calls in :func:`glob.glob` and :func:`~glob.iglob`, - thereby improving the speed of globbing operations by 20-80%. - (Contributed by Barney Gale in :gh:`116380`.) - - io --- * :mod:`io` which provides the built-in :func:`open` makes less system calls diff --git a/Lib/glob.py b/Lib/glob.py index 7ac497d4f0075a..d1a6dddeeb1610 100644 --- a/Lib/glob.py +++ b/Lib/glob.py @@ -1,10 +1,13 @@ """Filename globbing utility.""" +import contextlib import os import re import fnmatch import functools +import itertools import operator +import stat import sys @@ -42,35 +45,82 @@ def iglob(pathname, *, root_dir=None, dir_fd=None, recursive=False, """ sys.audit("glob.glob", pathname, recursive) sys.audit("glob.glob/2", pathname, recursive, root_dir, dir_fd) - pathname = os.fspath(pathname) - if isinstance(pathname, bytes): - pathname = os.fsdecode(pathname) - if root_dir is not None: - root_dir = os.fsdecode(root_dir) - paths = _iglob(pathname, root_dir, dir_fd, recursive, include_hidden) - return map(os.fsencode, paths) + if root_dir is not None: + root_dir = os.fspath(root_dir) + else: + root_dir = pathname[:0] + it = _iglob(pathname, root_dir, dir_fd, recursive, False, + include_hidden=include_hidden) + if not pathname or recursive and _isrecursive(pathname[:2]): + try: + s = next(it) # skip empty string + if s: + it = itertools.chain((s,), it) + except StopIteration: + pass + return it + +def _iglob(pathname, root_dir, dir_fd, recursive, dironly, + include_hidden=False): + dirname, basename = os.path.split(pathname) + if not has_magic(pathname): + assert not dironly + if basename: + if _lexists(_join(root_dir, pathname), dir_fd): + yield pathname + else: + # Patterns ending with a slash should match only directories + if _isdir(_join(root_dir, dirname), dir_fd): + yield pathname + return + if not dirname: + if recursive and _isrecursive(basename): + yield from _glob2(root_dir, basename, dir_fd, dironly, + include_hidden=include_hidden) + else: + yield from _glob1(root_dir, basename, dir_fd, dironly, + include_hidden=include_hidden) + return + # `os.path.split()` returns the argument itself as a dirname if it is a + # drive or UNC path. Prevent an infinite recursion if a drive or UNC path + # contains magic characters (i.e. r'\\?\C:'). + if dirname != pathname and has_magic(dirname): + dirs = _iglob(dirname, root_dir, dir_fd, recursive, True, + include_hidden=include_hidden) + else: + dirs = [dirname] + if has_magic(basename): + if recursive and _isrecursive(basename): + glob_in_dir = _glob2 + else: + glob_in_dir = _glob1 else: - return _iglob(pathname, root_dir, dir_fd, recursive, include_hidden) - -def _iglob(pathname, root_dir, dir_fd, recursive, include_hidden): - if os.path.altsep: - pathname = pathname.replace(os.path.altsep, os.path.sep) - drive, root, tail = os.path.splitroot(pathname) - parts = tail.split(os.path.sep)[::-1] if tail else [] - globber = _StringGlobber(recursive=recursive, include_hidden=include_hidden) - select = globber.selector(parts) - if drive: - root = drive + root - return select(root, dir_fd, root) - elif root: - return select(root, dir_fd, root, exists=True) - elif not root_dir: - return select(root, dir_fd, root, empty=True) + glob_in_dir = _glob0 + for dirname in dirs: + for name in glob_in_dir(_join(root_dir, dirname), basename, dir_fd, dironly, + include_hidden=include_hidden): + yield os.path.join(dirname, name) + +# These 2 helper functions non-recursively glob inside a literal directory. +# They return a list of basenames. _glob1 accepts a pattern while _glob0 +# takes a literal basename (so it only has to check for its existence). + +def _glob1(dirname, pattern, dir_fd, dironly, include_hidden=False): + names = _listdir(dirname, dir_fd, dironly) + if not (include_hidden or _ishidden(pattern)): + names = (x for x in names if not _ishidden(x)) + return fnmatch.filter(names, pattern) + +def _glob0(dirname, basename, dir_fd, dironly, include_hidden=False): + if basename: + if _lexists(_join(dirname, basename), dir_fd): + return [basename] else: - root = os.path.join(root_dir, '') - root_len = len(root) - paths = select(root, dir_fd, root, empty=True) - return (path[root_len:] for path in paths) + # `os.path.split()` returns an empty basename for paths ending with a + # directory separator. 'q*x/' should match only directories. + if _isdir(dirname, dir_fd): + return [basename] + return [] _deprecated_function_message = ( "{name} is deprecated and will be removed in Python {remove}. Use " @@ -80,16 +130,102 @@ def _iglob(pathname, root_dir, dir_fd, recursive, include_hidden): def glob0(dirname, pattern): import warnings warnings._deprecated("glob.glob0", _deprecated_function_message, remove=(3, 15)) - dirname = os.path.join(dirname, '') - select = _StringGlobber().literal_selector(pattern, []) - return [path[len(dirname):] for path in select(dirname)] + return _glob0(dirname, pattern, None, False) def glob1(dirname, pattern): import warnings warnings._deprecated("glob.glob1", _deprecated_function_message, remove=(3, 15)) - dirname = os.path.join(dirname, '') - select = _StringGlobber().wildcard_selector(pattern, []) - return [path[len(dirname):] for path in select(dirname)] + return _glob1(dirname, pattern, None, False) + +# This helper function recursively yields relative pathnames inside a literal +# directory. + +def _glob2(dirname, pattern, dir_fd, dironly, include_hidden=False): + assert _isrecursive(pattern) + if not dirname or _isdir(dirname, dir_fd): + yield pattern[:0] + yield from _rlistdir(dirname, dir_fd, dironly, + include_hidden=include_hidden) + +# If dironly is false, yields all file names inside a directory. +# If dironly is true, yields only directory names. +def _iterdir(dirname, dir_fd, dironly): + try: + fd = None + fsencode = None + if dir_fd is not None: + if dirname: + fd = arg = os.open(dirname, _dir_open_flags, dir_fd=dir_fd) + else: + arg = dir_fd + if isinstance(dirname, bytes): + fsencode = os.fsencode + elif dirname: + arg = dirname + elif isinstance(dirname, bytes): + arg = bytes(os.curdir, 'ASCII') + else: + arg = os.curdir + try: + with os.scandir(arg) as it: + for entry in it: + try: + if not dironly or entry.is_dir(): + if fsencode is not None: + yield fsencode(entry.name) + else: + yield entry.name + except OSError: + pass + finally: + if fd is not None: + os.close(fd) + except OSError: + return + +def _listdir(dirname, dir_fd, dironly): + with contextlib.closing(_iterdir(dirname, dir_fd, dironly)) as it: + return list(it) + +# Recursively yields relative pathnames inside a literal directory. +def _rlistdir(dirname, dir_fd, dironly, include_hidden=False): + names = _listdir(dirname, dir_fd, dironly) + for x in names: + if include_hidden or not _ishidden(x): + yield x + path = _join(dirname, x) if dirname else x + for y in _rlistdir(path, dir_fd, dironly, + include_hidden=include_hidden): + yield _join(x, y) + + +def _lexists(pathname, dir_fd): + # Same as os.path.lexists(), but with dir_fd + if dir_fd is None: + return os.path.lexists(pathname) + try: + os.lstat(pathname, dir_fd=dir_fd) + except (OSError, ValueError): + return False + else: + return True + +def _isdir(pathname, dir_fd): + # Same as os.path.isdir(), but with dir_fd + if dir_fd is None: + return os.path.isdir(pathname) + try: + st = os.stat(pathname, dir_fd=dir_fd) + except (OSError, ValueError): + return False + else: + return stat.S_ISDIR(st.st_mode) + +def _join(dirname, basename): + # It is common if dirname or basename is empty + if not dirname or not basename: + return dirname or basename + return os.path.join(dirname, basename) magic_check = re.compile('([*?[])') magic_check_bytes = re.compile(b'([*?[])') @@ -101,6 +237,15 @@ def has_magic(s): match = magic_check.search(s) return match is not None +def _ishidden(path): + return path[0] in ('.', b'.'[0]) + +def _isrecursive(pattern): + if isinstance(pattern, bytes): + return pattern == b'**' + else: + return pattern == '**' + def escape(pathname): """Escape all special characters. """ @@ -174,13 +319,12 @@ def translate(pat, *, recursive=False, include_hidden=False, seps=None): return fr'(?s:{res})\Z' -(a)functools.lru_cache(maxsize=1024) -def _compile_pattern(pat, sep, case_sensitive, recursive, include_hidden): +(a)functools.lru_cache(maxsize=512) +def _compile_pattern(pat, sep, case_sensitive, recursive=True): """Compile given glob pattern to a re.Pattern object (observing case sensitivity).""" flags = re.NOFLAG if case_sensitive else re.IGNORECASE - regex = translate(pat, recursive=recursive, - include_hidden=include_hidden, seps=sep) + regex = translate(pat, recursive=recursive, include_hidden=True, seps=sep) return re.compile(regex, flags=flags).match @@ -188,13 +332,11 @@ class _GlobberBase: """Abstract class providing shell-style pattern matching and globbing. """ - def __init__(self, sep=os.path.sep, case_sensitive=os.name != 'nt', - case_pedantic=False, recursive=False, include_hidden=False): + def __init__(self, sep, case_sensitive, case_pedantic=False, recursive=False): self.sep = sep self.case_sensitive = case_sensitive self.case_pedantic = case_pedantic self.recursive = recursive - self.include_hidden = include_hidden # Abstract methods @@ -204,38 +346,12 @@ def lexists(path): """ raise NotImplementedError - @staticmethod - def lstat(path, dir_fd=None): - """Implements os.lstat() - """ - raise NotImplementedError - - @staticmethod - def open(path, flags, dir_fd=None): - """Implements os.open() - """ - raise NotImplementedError - @staticmethod def scandir(path): """Like os.scandir(), but generates (entry, name, path) tuples. """ raise NotImplementedError - @staticmethod - def scandir_cwd(): - raise NotImplementedError - - @staticmethod - def scandir_fd(fd, prefix): - raise NotImplementedError - - @staticmethod - def close(fd): - """Implements os.close(). - """ - raise NotImplementedError - @staticmethod def concat_path(path, text): """Implements path concatenation. @@ -245,8 +361,7 @@ def concat_path(path, text): # High-level methods def compile(self, pat): - return _compile_pattern(pat, self.sep, self.case_sensitive, - self.recursive, self.include_hidden) + return _compile_pattern(pat, self.sep, self.case_sensitive, self.recursive) def selector(self, parts): """Returns a function that selects from a given path, walking and @@ -271,14 +386,10 @@ def special_selector(self, part, parts): if parts: part += self.sep select_next = self.selector(parts) - if not part: - return select_next - def select_special(path, dir_fd=None, rel_path=None, exists=False, empty=False): + def select_special(path, exists=False): path = self.concat_path(path, part) - if dir_fd is not None: - rel_path = self.concat_path(rel_path, part) - return select_next(path, dir_fd, rel_path, exists) + return select_next(path, exists) return select_special def literal_selector(self, part, parts): @@ -295,11 +406,9 @@ def literal_selector(self, part, parts): select_next = self.selector(parts) - def select_literal(path, dir_fd=None, rel_path=None, exists=False, empty=False): + def select_literal(path, exists=False): path = self.concat_path(path, part) - if dir_fd is not None: - rel_path = self.concat_path(rel_path, part) - return select_next(path, dir_fd, rel_path) + return select_next(path, exists=False) return select_literal def wildcard_selector(self, part, parts): @@ -307,24 +416,14 @@ def wildcard_selector(self, part, parts): filtering by pattern. """ - match = None if self.include_hidden and part == '*' else self.compile(part) + match = None if part == '*' else self.compile(part) dir_only = bool(parts) if dir_only: select_next = self.selector(parts) - def select_wildcard(path, dir_fd=None, rel_path=None, exists=False, empty=False): - close_fd = False + def select_wildcard(path, exists=False): try: - if dir_fd is None: - fd = None - entries = self.scandir(path) if path else self.scandir_cwd() - elif not rel_path: - fd = dir_fd - entries = self.scandir_fd(fd, path) - else: - fd = self.open(rel_path, _dir_open_flags, dir_fd=dir_fd) - close_fd = True - entries = self.scandir_fd(fd, path) + entries = self.scandir(path) except OSError: pass else: @@ -337,17 +436,9 @@ def select_wildcard(path, dir_fd=None, rel_path=None, exists=False, empty=False) except OSError: continue entry_path = self.concat_path(entry_path, self.sep) - if fd is not None: - entry_name = entry_name + self.sep - yield from select_next( - entry_path, fd, entry_name, exists=True) + yield from select_next(entry_path, exists=True) else: - # Optimization: directly yield the path if this is - # last pattern part. yield entry_path - finally: - if close_fd: - self.close(fd) return select_wildcard def recursive_selector(self, part, parts): @@ -369,49 +460,26 @@ def recursive_selector(self, part, parts): while parts and parts[-1] not in _special_parts: part += self.sep + parts.pop() - match = None if self.include_hidden and part == '**' else self.compile(part) + match = None if part == '**' else self.compile(part) dir_only = bool(parts) select_next = self.selector(parts) - def select_recursive(path, dir_fd=None, rel_path=None, exists=False, empty=False): + def select_recursive(path, exists=False): match_pos = len(str(path)) if match is None or match(str(path), match_pos): - yield from select_next(path, dir_fd, rel_path, exists, empty) - stack = [(path, dir_fd, rel_path)] - try: - while stack: - yield from select_recursive_step(stack, match_pos) - finally: - # Close any file descriptors still on the stack. - while stack: - path, dir_fd, _rel_path = stack.pop() - if path is None: - try: - self.close(dir_fd) - except OSError: - pass + yield from select_next(path, exists) + stack = [path] + while stack: + yield from select_recursive_step(stack, match_pos) def select_recursive_step(stack, match_pos): - path, dir_fd, rel_path = stack.pop() + path = stack.pop() try: - if path is None: - self.close(dir_fd) - return - elif dir_fd is None: - fd = None - entries = self.scandir(path) if path else self.scandir_cwd() - elif not rel_path: - fd = dir_fd - entries = self.scandir_fd(fd, path) - else: - fd = self.open(rel_path, _dir_open_flags, dir_fd=dir_fd) - # Schedule the file descriptor to be closed next step. - stack.append((None, fd, None)) - entries = self.scandir_fd(fd, path) + entries = self.scandir(path) except OSError: pass else: - for entry, entry_name, entry_path in entries: + for entry, _entry_name, entry_path in entries: is_dir = False try: if entry.is_dir(follow_symlinks=follow_symlinks): @@ -423,38 +491,25 @@ def select_recursive_step(stack, match_pos): entry_path_str = str(entry_path) if dir_only: entry_path = self.concat_path(entry_path, self.sep) - if fd is not None: - entry_name = entry_name + self.sep if match is None or match(entry_path_str, match_pos): if dir_only: - yield from select_next( - entry_path, fd, entry_name, exists=True) + yield from select_next(entry_path, exists=True) else: # Optimization: directly yield the path if this is # last pattern part. yield entry_path if is_dir: - stack.append((entry_path, fd, entry_name)) + stack.append(entry_path) return select_recursive - def select_exists(self, path, dir_fd=None, rel_path=None, exists=False, empty=False): - """Yields the given path, if it exists. If *dir_fd* is given, we check - whether *rel_path* exists relative to the fd. + def select_exists(self, path, exists=False): + """Yields the given path, if it exists. """ - if empty: - # Suppress initial path so iglob() doesn't yield the empty string. - pass - elif exists: + if exists: # Optimization: this path is already known to exist, e.g. because # it was returned from os.scandir(), so we skip calling lstat(). yield path - elif dir_fd is not None: - try: - self.lstat(rel_path, dir_fd=dir_fd) - yield path - except OSError: - pass elif self.lexists(path): yield path @@ -463,9 +518,6 @@ class _StringGlobber(_GlobberBase): """Provides shell-style pattern matching and globbing for string paths. """ lexists = staticmethod(os.path.lexists) - lstat = staticmethod(os.lstat) - open = staticmethod(os.open) - close = staticmethod(os.close) concat_path = operator.add @staticmethod @@ -476,20 +528,6 @@ def scandir(path): entries = list(scandir_it) return ((entry, entry.name, entry.path) for entry in entries) - @staticmethod - def scandir_cwd(): - with os.scandir() as scandir_it: - entries = list(scandir_it) - # Suppress leading dot when scanning current directory. - return ((entry, entry.name, entry.name) for entry in entries) - - @staticmethod - def scandir_fd(fd, prefix): - prefix = os.path.join(prefix, prefix[:0]) - with os.scandir(fd) as scandir_it: - entries = list(scandir_it) - return ((entry, entry.name, prefix + entry.name) for entry in entries) - class _PathGlobber(_GlobberBase): """Provides shell-style pattern matching and globbing for pathlib paths. diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index bd2252ef77dcf6..930701d4789f5c 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -196,8 +196,7 @@ def full_match(self, pattern, *, case_sensitive=None): pattern = self.with_segments(pattern) if case_sensitive is None: case_sensitive = self.parser.normcase('Aa') == 'Aa' - globber = _PathGlobber(pattern.parser.sep, case_sensitive, - recursive=True, include_hidden=True) + globber = _PathGlobber(pattern.parser.sep, case_sensitive, recursive=True) match = globber.compile(str(pattern)) return match(str(self)) is not None diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index 492bae0b7f7cd8..83da6960f00489 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -558,8 +558,7 @@ def full_match(self, pattern, *, case_sensitive=None): # paths shouldn't match wildcards, so we change it to the empty string. path = str(self) if self.parts else '' pattern = str(pattern) if pattern.parts else '' - globber = _StringGlobber(self.parser.sep, case_sensitive, - recursive=True, include_hidden=True) + globber = _StringGlobber(self.parser.sep, case_sensitive, recursive=True) return globber.compile(pattern)(path) is not None def match(self, path_pattern, *, case_sensitive=None): @@ -581,8 +580,7 @@ def match(self, path_pattern, *, case_sensitive=None): return False if len(path_parts) > len(pattern_parts) and path_pattern.anchor: return False - globber = _StringGlobber(self.parser.sep, case_sensitive, - include_hidden=True) + globber = _StringGlobber(self.parser.sep, case_sensitive) for path_part, pattern_part in zip(path_parts, pattern_parts): match = globber.compile(pattern_part) if match(path_part) is None: @@ -857,8 +855,7 @@ def glob(self, pattern, *, case_sensitive=None, recurse_symlinks=False): case_pedantic = True parts = self._parse_pattern(pattern) recursive = True if recurse_symlinks else _no_recurse_symlinks - globber = _StringGlobber(self.parser.sep, case_sensitive, case_pedantic, - recursive, include_hidden=True) + globber = _StringGlobber(self.parser.sep, case_sensitive, case_pedantic, recursive) select = globber.selector(parts[::-1]) root = str(self) paths = select(self.parser.join(root, '')) diff --git a/Lib/test/test_glob.py b/Lib/test/test_glob.py index 2290e06e60e377..da73769c16e9af 100644 --- a/Lib/test/test_glob.py +++ b/Lib/test/test_glob.py @@ -4,18 +4,14 @@ import shutil import sys import unittest -import unittest.mock import warnings from test import support -from test.support import is_wasi, Py_DEBUG, infinite_recursion +from test.support import is_wasi, Py_DEBUG from test.support.os_helper import (TESTFN, skip_unless_symlink, can_symlink, create_empty_file, change_cwd) -_supports_dir_fd = {os.open, os.stat} <= os.supports_dir_fd and os.scandir in os.supports_fd - - class GlobTests(unittest.TestCase): dir_fd = None @@ -53,7 +49,7 @@ def setUp(self): def open_dirfd(self): if self.dir_fd is not None: os.close(self.dir_fd) - if _supports_dir_fd: + if {os.open, os.stat} <= os.supports_dir_fd and os.scandir in os.supports_fd: self.dir_fd = os.open(self.tempdir, os.O_RDONLY | os.O_DIRECTORY) else: self.dir_fd = None @@ -181,18 +177,20 @@ def test_glob_directory_with_trailing_slash(self): self.assertEqual(glob.glob(self.norm('Z*Z') + sep), []) self.assertEqual(glob.glob(self.norm('ZZZ') + sep), []) self.assertEqual(glob.glob(self.norm('aaa') + sep), - [self.norm('aaa') + os.sep]) - # Redundant separators are preserved and normalized + [self.norm('aaa') + sep]) + # Preserving the redundant separators is an implementation detail. self.assertEqual(glob.glob(self.norm('aaa') + sep*2), - [self.norm('aaa') + os.sep*2]) + [self.norm('aaa') + sep*2]) # When there is a wildcard pattern which ends with a pathname # separator, glob() doesn't blow. # The result should end with the pathname separator. + # Normalizing the trailing separator is an implementation detail. eq = self.assertSequencesEqual_noorder eq(glob.glob(self.norm('aa*') + sep), [self.norm('aaa') + os.sep, self.norm('aab') + os.sep]) + # Stripping the redundant separators is an implementation detail. eq(glob.glob(self.norm('aa*') + sep*2), - [self.norm('aaa') + os.sep*2, self.norm('aab') + os.sep*2]) + [self.norm('aaa') + os.sep, self.norm('aab') + os.sep]) def test_glob_bytes_directory_with_trailing_slash(self): # Same as test_glob_directory_with_trailing_slash, but with a @@ -202,16 +200,16 @@ def test_glob_bytes_directory_with_trailing_slash(self): self.assertEqual(glob.glob(os.fsencode(self.norm('Z*Z') + sep)), []) self.assertEqual(glob.glob(os.fsencode(self.norm('ZZZ') + sep)), []) self.assertEqual(glob.glob(os.fsencode(self.norm('aaa') + sep)), - [os.fsencode(self.norm('aaa') + os.sep)]) + [os.fsencode(self.norm('aaa') + sep)]) self.assertEqual(glob.glob(os.fsencode(self.norm('aaa') + sep*2)), - [os.fsencode(self.norm('aaa') + os.sep*2)]) + [os.fsencode(self.norm('aaa') + sep*2)]) eq = self.assertSequencesEqual_noorder eq(glob.glob(os.fsencode(self.norm('aa*') + sep)), [os.fsencode(self.norm('aaa') + os.sep), os.fsencode(self.norm('aab') + os.sep)]) eq(glob.glob(os.fsencode(self.norm('aa*') + sep*2)), - [os.fsencode(self.norm('aaa') + os.sep*2), - os.fsencode(self.norm('aab') + os.sep*2)]) + [os.fsencode(self.norm('aaa') + os.sep), + os.fsencode(self.norm('aab') + os.sep)]) @skip_unless_symlink def test_glob_symlinks(self): @@ -328,12 +326,8 @@ def test_recursive_glob(self): with change_cwd(self.tempdir): join = os.path.join eq(glob.glob('**', recursive=True), [join(*i) for i in full]) - eq(glob.glob(join('**', '**'), recursive=True), - [join(*i) for i in full]) eq(glob.glob(join('**', ''), recursive=True), [join(*i) for i in dirs]) - eq(glob.glob(join('**', '**', ''), recursive=True), - [join(*i) for i in dirs]) eq(glob.glob(join('**', '*'), recursive=True), [join(*i) for i in full]) eq(glob.glob(join(os.curdir, '**'), recursive=True), @@ -400,33 +394,6 @@ def test_glob_many_open_files(self): for it in iters: self.assertEqual(next(it), p) - def test_glob_above_recursion_limit(self): - depth = 30 - base = os.path.join(self.tempdir, 'deep') - p = os.path.join(base, *(['d']*depth)) - os.makedirs(p) - pattern = os.path.join(base, '**', 'd') - with infinite_recursion(depth - 5): - glob.glob(pattern, recursive=True) - - @unittest.skipUnless(_supports_dir_fd, "Needs support for iglob(dir_fd=...)") - def test_iglob_iter_close(self): - base = os.path.join(self.tempdir, 'deep') - p = os.path.join(base, *(['d'] * 10)) - os.makedirs(p) - with ( - unittest.mock.patch("glob._StringGlobber.open", wraps=os.open) as os_open, - unittest.mock.patch("glob._StringGlobber.close", wraps=os.close) as os_close - ): - self.assertEqual(os_open.call_count, os_close.call_count) - iter = glob.iglob('**/*/d', dir_fd=self.dir_fd, recursive=True) - self.assertEqual(os_open.call_count, os_close.call_count) - self.assertEqual(next(iter), 'deep/d') - self.assertEqual(next(iter), 'deep/d/d') - self.assertGreater(os_open.call_count, os_close.call_count) - iter.close() - self.assertEqual(os_open.call_count, os_close.call_count) - def test_glob0(self): with self.assertWarns(DeprecationWarning): glob.glob0(self.tempdir, 'a') diff --git a/Misc/NEWS.d/next/Library/2024-03-05-23-08-11.gh-issue-116380.56HU7I.rst b/Misc/NEWS.d/next/Library/2024-03-05-23-08-11.gh-issue-116380.56HU7I.rst deleted file mode 100644 index b7f27ab7191a96..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-03-05-23-08-11.gh-issue-116380.56HU7I.rst +++ /dev/null @@ -1,2 +0,0 @@ -Speed up :func:`glob.glob` and :func:`glob.iglob` by making use of -:func:`glob.translate` and tracking path existence more precisely.

1 0

[3.12] gh-130618: Fix parser error when using lambdas inside f-strings (GH-130638) (#130644)
by pablogsal March 1, 2025

March 1, 2025

https://github.com/python/cpython/commit/0860d9c72fdf72f8dc7acde832a4e61532… commit: 0860d9c72fdf72f8dc7acde832a4e61532f2caf7 branch: 3.12 author: Pablo Galindo Salgado <Pablogsal(a)gmail.com> committer: pablogsal <Pablogsal(a)gmail.com> date: 2025-03-01T18:10:07Z summary: [3.12] gh-130618: Fix parser error when using lambdas inside f-strings (GH-130638) (#130644) (cherry picked from commit e06bebb87e1b33f7251196e1ddb566f528c3fc98) files: A Misc/NEWS.d/next/Core_and_Builtins/2025-02-27-15-07-06.gh-issue-130618.JTcsRB.rst M Lib/test/test_grammar.py M Parser/tokenizer.c diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index c72f4387108ca8..46eb5736fe3a89 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -2030,6 +2030,18 @@ async def foo(): with self.assertRaises(Done): foo().send(None) + def test_complex_lambda(self): + def test1(foo, bar): + return "" + + def test2(): + return f"{test1( + foo=lambda: '、、、、、、、、、、、、、、、、、', + bar=lambda: 'abcdefghijklmnopqrstuvwxyz 123456789 123456789', + )}" + + self.assertEqual(test2(), "") + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-02-27-15-07-06.gh-issue-130618.JTcsRB.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-02-27-15-07-06.gh-issue-130618.JTcsRB.rst new file mode 100644 index 00000000000000..de67496108e2a7 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-02-27-15-07-06.gh-issue-130618.JTcsRB.rst @@ -0,0 +1,3 @@ +Fix a bug that was causing ``UnicodeDecodeError`` or ``SystemError`` to be +raised when using f-strings with ``lambda`` expressions with non-ASCII +characters. Patch by Pablo Galindo diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 7ba717d909124e..d2a0eaca701b78 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -479,8 +479,11 @@ static int update_fstring_expr(struct tok_state *tok, char cur) { break; case '}': case '!': - case ':': tok_mode->last_expr_end = strlen(tok->start); + case ':': + if (tok_mode->last_expr_end == -1) { + tok_mode->last_expr_end = strlen(tok->start); + } break; default: Py_UNREACHABLE();

1 0

[3.13] gh-130618: Fix parser error when using lambdas inside f-strings (GH-130638) (#130642)
by pablogsal March 1, 2025

March 1, 2025

https://github.com/python/cpython/commit/2d7145decb1acad4636a21b77ab3a9c4f4… commit: 2d7145decb1acad4636a21b77ab3a9c4f48af36d branch: 3.13 author: Pablo Galindo Salgado <Pablogsal(a)gmail.com> committer: pablogsal <Pablogsal(a)gmail.com> date: 2025-03-01T18:10:03Z summary: [3.13] gh-130618: Fix parser error when using lambdas inside f-strings (GH-130638) (#130642) (cherry picked from commit e06bebb87e1b33f7251196e1ddb566f528c3fc98) Signed-off-by: Pablo Galindo <pablogsal(a)gmail.com> files: A Misc/NEWS.d/next/Core_and_Builtins/2025-02-27-15-07-06.gh-issue-130618.JTcsRB.rst M Lib/test/test_grammar.py M Parser/lexer/lexer.c diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index c72f4387108ca8..46eb5736fe3a89 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -2030,6 +2030,18 @@ async def foo(): with self.assertRaises(Done): foo().send(None) + def test_complex_lambda(self): + def test1(foo, bar): + return "" + + def test2(): + return f"{test1( + foo=lambda: '、、、、、、、、、、、、、、、、、', + bar=lambda: 'abcdefghijklmnopqrstuvwxyz 123456789 123456789', + )}" + + self.assertEqual(test2(), "") + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-02-27-15-07-06.gh-issue-130618.JTcsRB.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-02-27-15-07-06.gh-issue-130618.JTcsRB.rst new file mode 100644 index 00000000000000..de67496108e2a7 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-02-27-15-07-06.gh-issue-130618.JTcsRB.rst @@ -0,0 +1,3 @@ +Fix a bug that was causing ``UnicodeDecodeError`` or ``SystemError`` to be +raised when using f-strings with ``lambda`` expressions with non-ASCII +characters. Patch by Pablo Galindo diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 3ced1be0b003b8..15e6332ee35718 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -211,9 +211,13 @@ _PyLexer_update_fstring_expr(struct tok_state *tok, char cur) break; case '}': case '!': - case ':': tok_mode->last_expr_end = strlen(tok->start); break; + case ':': + if (tok_mode->last_expr_end == -1) { + tok_mode->last_expr_end = strlen(tok->start); + } + break; default: Py_UNREACHABLE(); }

1 0

[3.13] Revert "gh-128364: Fix flaky `test_timeout` test (gh-130724)" (gh-130732) (#130734)
by colesbury March 1, 2025

March 1, 2025

https://github.com/python/cpython/commit/a0ea87c0c3b500f27ee201955be48e2157… commit: a0ea87c0c3b500f27ee201955be48e2157a0a471 branch: 3.13 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: colesbury <colesbury(a)gmail.com> date: 2025-03-01T17:44:50Z summary: [3.13] Revert "gh-128364: Fix flaky `test_timeout` test (gh-130724)" (gh-130732) (#130734) Revert "gh-128364: Fix flaky `test_timeout` test (gh-130724)" (gh-130732) Change broke Android and iOS buildbots that do not have multiprocessing. This reverts commit cfa0b1dc375e63cde28e61a47108c645b0e74834. (cherry picked from commit 5221d9ce0e2beb0fe04bed072e1bb448fd522882) Co-authored-by: Sam Gross <colesbury(a)gmail.com> files: M Lib/test/test_concurrent_futures/test_wait.py M Lib/test/test_concurrent_futures/util.py diff --git a/Lib/test/test_concurrent_futures/test_wait.py b/Lib/test/test_concurrent_futures/test_wait.py index 697285779510a9..108cf54bf79e6f 100644 --- a/Lib/test/test_concurrent_futures/test_wait.py +++ b/Lib/test/test_concurrent_futures/test_wait.py @@ -114,8 +114,9 @@ def test_all_completed(self): def test_timeout(self): short_timeout = 0.050 + long_timeout = short_timeout * 10 - future = self.executor.submit(self.event.wait) + future = self.executor.submit(time.sleep, long_timeout) finished, pending = futures.wait( [CANCELLED_AND_NOTIFIED_FUTURE, @@ -131,9 +132,6 @@ def test_timeout(self): finished) self.assertEqual(set([future]), pending) - # Set the event to allow the future to complete - self.event.set() - class ThreadPoolWaitTests(ThreadPoolMixin, WaitTests, BaseTestCase): diff --git a/Lib/test/test_concurrent_futures/util.py b/Lib/test/test_concurrent_futures/util.py index daadb8459c3161..3b8ec3e205d5aa 100644 --- a/Lib/test/test_concurrent_futures/util.py +++ b/Lib/test/test_concurrent_futures/util.py @@ -1,6 +1,5 @@ import multiprocessing import sys -import threading import time import unittest from concurrent import futures @@ -47,14 +46,11 @@ def setUp(self): self.t1 = time.monotonic() if hasattr(self, "ctx"): - self.manager = multiprocessing.Manager() - self.event = self.manager.Event() self.executor = self.executor_type( max_workers=self.worker_count, mp_context=self.get_context(), **self.executor_kwargs) else: - self.event = threading.Event() self.executor = self.executor_type( max_workers=self.worker_count, **self.executor_kwargs) @@ -62,9 +58,6 @@ def setUp(self): def tearDown(self): self.executor.shutdown(wait=True) self.executor = None - if hasattr(self, "ctx"): - self.manager.shutdown() - self.manager = None dt = time.monotonic() - self.t1 if support.verbose:

1 0

[3.12] Revert "gh-128364: Fix flaky `test_timeout` test (gh-130724)" (gh-130732) (#130735)
by colesbury March 1, 2025

March 1, 2025

https://github.com/python/cpython/commit/99e18abe6f3b3a0fcb6b19381e2e77c8af… commit: 99e18abe6f3b3a0fcb6b19381e2e77c8afaf007e branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: colesbury <colesbury(a)gmail.com> date: 2025-03-01T17:40:51Z summary: [3.12] Revert "gh-128364: Fix flaky `test_timeout` test (gh-130724)" (gh-130732) (#130735) Revert "gh-128364: Fix flaky `test_timeout` test (gh-130724)" (gh-130732) Change broke Android and iOS buildbots that do not have multiprocessing. This reverts commit cfa0b1dc375e63cde28e61a47108c645b0e74834. (cherry picked from commit 5221d9ce0e2beb0fe04bed072e1bb448fd522882) Co-authored-by: Sam Gross <colesbury(a)gmail.com> files: M Lib/test/test_concurrent_futures/test_wait.py M Lib/test/test_concurrent_futures/util.py diff --git a/Lib/test/test_concurrent_futures/test_wait.py b/Lib/test/test_concurrent_futures/test_wait.py index 3527a74a708b20..ff486202092c81 100644 --- a/Lib/test/test_concurrent_futures/test_wait.py +++ b/Lib/test/test_concurrent_futures/test_wait.py @@ -114,8 +114,9 @@ def test_all_completed(self): def test_timeout(self): short_timeout = 0.050 + long_timeout = short_timeout * 10 - future = self.executor.submit(self.event.wait) + future = self.executor.submit(time.sleep, long_timeout) finished, pending = futures.wait( [CANCELLED_AND_NOTIFIED_FUTURE, @@ -131,9 +132,6 @@ def test_timeout(self): finished) self.assertEqual(set([future]), pending) - # Set the event to allow the future to complete - self.event.set() - class ThreadPoolWaitTests(ThreadPoolMixin, WaitTests, BaseTestCase): diff --git a/Lib/test/test_concurrent_futures/util.py b/Lib/test/test_concurrent_futures/util.py index 93ae2459ac5237..fc6030e375fb6b 100644 --- a/Lib/test/test_concurrent_futures/util.py +++ b/Lib/test/test_concurrent_futures/util.py @@ -1,6 +1,5 @@ import multiprocessing import sys -import threading import time import unittest from concurrent import futures @@ -47,14 +46,11 @@ def setUp(self): self.t1 = time.monotonic() if hasattr(self, "ctx"): - self.manager = multiprocessing.Manager() - self.event = self.manager.Event() self.executor = self.executor_type( max_workers=self.worker_count, mp_context=self.get_context(), **self.executor_kwargs) else: - self.event = threading.Event() self.executor = self.executor_type( max_workers=self.worker_count, **self.executor_kwargs) @@ -62,9 +58,6 @@ def setUp(self): def tearDown(self): self.executor.shutdown(wait=True) self.executor = None - if hasattr(self, "ctx"): - self.manager.shutdown() - self.manager = None dt = time.monotonic() - self.t1 if support.verbose:

1 0