[Python-checkins] cpython: Issue #25596: Optimized glob() and iglob() functions in the
serhiy.storchaka
python-checkins at python.org
Tue Sep 6 15:35:26 EDT 2016
https://hg.python.org/cpython/rev/cb7ee9d9cddd
changeset: 103132:cb7ee9d9cddd
parent: 103130:28e280915508
user: Serhiy Storchaka <storchaka at gmail.com>
date: Tue Sep 06 22:33:41 2016 +0300
summary:
Issue #25596: Optimized glob() and iglob() functions in the
glob module; they are now about 3--6 times faster.
files:
Doc/library/glob.rst | 2 +-
Doc/whatsnew/3.6.rst | 4 +
Lib/glob.py | 70 ++++++++++++++++++-------------
Misc/NEWS | 3 +
4 files changed, 49 insertions(+), 30 deletions(-)
diff --git a/Doc/library/glob.rst b/Doc/library/glob.rst
--- a/Doc/library/glob.rst
+++ b/Doc/library/glob.rst
@@ -14,7 +14,7 @@
according to the rules used by the Unix shell, although results are returned in
arbitrary order. No tilde expansion is done, but ``*``, ``?``, and character
ranges expressed with ``[]`` will be correctly matched. This is done by using
-the :func:`os.listdir` and :func:`fnmatch.fnmatch` functions in concert, and
+the :func:`os.scandir` and :func:`fnmatch.fnmatch` functions in concert, and
not by actually invoking a subshell. Note that unlike :func:`fnmatch.fnmatch`,
:mod:`glob` treats filenames beginning with a dot (``.``) as special cases.
(For tilde and shell variable expansion, use :func:`os.path.expanduser` and
diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst
--- a/Doc/whatsnew/3.6.rst
+++ b/Doc/whatsnew/3.6.rst
@@ -767,6 +767,10 @@
Argument Clinic this overhead is significantly decreased.
(Contributed by Serhiy Storchaka in :issue:`27574`).
+* Optimized :func:`~glob.glob` and :func:`~glob.iglob` functions in the
+ :mod:`glob` module; they are now about 3--6 times faster.
+ (Contributed by Serhiy Storchaka in :issue:`25596`).
+
Build and C API Changes
=======================
diff --git a/Lib/glob.py b/Lib/glob.py
--- a/Lib/glob.py
+++ b/Lib/glob.py
@@ -30,15 +30,16 @@
If recursive is true, the pattern '**' will match any files and
zero or more directories and subdirectories.
"""
- it = _iglob(pathname, recursive)
+ it = _iglob(pathname, recursive, False)
if recursive and _isrecursive(pathname):
s = next(it) # skip empty string
assert not s
return it
-def _iglob(pathname, recursive):
+def _iglob(pathname, recursive, dironly):
dirname, basename = os.path.split(pathname)
if not has_magic(pathname):
+ assert not dironly
if basename:
if os.path.lexists(pathname):
yield pathname
@@ -49,47 +50,39 @@
return
if not dirname:
if recursive and _isrecursive(basename):
- yield from glob2(dirname, basename)
+ yield from _glob2(dirname, basename, dironly)
else:
- yield from glob1(dirname, basename)
+ yield from _glob1(dirname, basename, dironly)
return
# `os.path.split()` returns the argument itself as a dirname if it is a
# drive or UNC path. Prevent an infinite recursion if a drive or UNC path
# contains magic characters (i.e. r'\\?\C:').
if dirname != pathname and has_magic(dirname):
- dirs = _iglob(dirname, recursive)
+ dirs = _iglob(dirname, recursive, True)
else:
dirs = [dirname]
if has_magic(basename):
if recursive and _isrecursive(basename):
- glob_in_dir = glob2
+ glob_in_dir = _glob2
else:
- glob_in_dir = glob1
+ glob_in_dir = _glob1
else:
- glob_in_dir = glob0
+ glob_in_dir = _glob0
for dirname in dirs:
- for name in glob_in_dir(dirname, basename):
+ for name in glob_in_dir(dirname, basename, dironly):
yield os.path.join(dirname, name)
# These 2 helper functions non-recursively glob inside a literal directory.
-# They return a list of basenames. `glob1` accepts a pattern while `glob0`
+# They return a list of basenames. _glob1 accepts a pattern while _glob0
# takes a literal basename (so it only has to check for its existence).
-def glob1(dirname, pattern):
- if not dirname:
- if isinstance(pattern, bytes):
- dirname = bytes(os.curdir, 'ASCII')
- else:
- dirname = os.curdir
- try:
- names = os.listdir(dirname)
- except OSError:
- return []
+def _glob1(dirname, pattern, dironly):
+ names = list(_iterdir(dirname, dironly))
if not _ishidden(pattern):
- names = [x for x in names if not _ishidden(x)]
+ names = (x for x in names if not _ishidden(x))
return fnmatch.filter(names, pattern)
-def glob0(dirname, basename):
+def _glob0(dirname, basename, dironly):
if not basename:
# `os.path.split()` returns an empty basename for paths ending with a
# directory separator. 'q*x/' should match only directories.
@@ -100,30 +93,49 @@
return [basename]
return []
+# Following functions are not public but can be used by third-party code.
+
+def glob0(dirname, pattern):
+ return _glob0(dirname, pattern, False)
+
+def glob1(dirname, pattern):
+ return _glob1(dirname, pattern, False)
+
# This helper function recursively yields relative pathnames inside a literal
# directory.
-def glob2(dirname, pattern):
+def _glob2(dirname, pattern, dironly):
assert _isrecursive(pattern)
yield pattern[:0]
- yield from _rlistdir(dirname)
+ yield from _rlistdir(dirname, dironly)
-# Recursively yields relative pathnames inside a literal directory.
-def _rlistdir(dirname):
+# If dironly is false, yields all file names inside a directory.
+# If dironly is true, yields only directory names.
+def _iterdir(dirname, dironly):
if not dirname:
if isinstance(dirname, bytes):
dirname = bytes(os.curdir, 'ASCII')
else:
dirname = os.curdir
try:
- names = os.listdir(dirname)
- except os.error:
+ with os.scandir(dirname) as it:
+ for entry in it:
+ try:
+ if not dironly or entry.is_dir():
+ yield entry.name
+ except OSError:
+ pass
+ except OSError:
return
+
+# Recursively yields relative pathnames inside a literal directory.
+def _rlistdir(dirname, dironly):
+ names = list(_iterdir(dirname, dironly))
for x in names:
if not _ishidden(x):
yield x
path = os.path.join(dirname, x) if dirname else x
- for y in _rlistdir(path):
+ for y in _rlistdir(path, dironly):
yield os.path.join(x, y)
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -89,6 +89,9 @@
Library
-------
+- Issue #25596: Optimized glob() and iglob() functions in the
+ glob module; they are now about 3--6 times faster.
+
- Issue #27928: Add scrypt (password-based key derivation function) to
hashlib module (requires OpenSSL 1.1.0).
--
Repository URL: https://hg.python.org/cpython
More information about the Python-checkins
mailing list