waling a directory with very many files

[Nick Craig-Wood]

tom <fsb at thefsb.org> wrote:
>  On Jun 14, 1:35 pm, Tim Golden <m... at timgolden.me.uk> wrote:
> >
> > If you're on Windows, you can use the win32file.FindFilesIterator
> > function from the pywin32 package. (Which wraps the Win32 API
> > FindFirstFile / FindNextFile pattern).
> 
>  thanks, tim.
> 
>  however, i'm not using windows. freebsd and os x.

Here is a ctypes generator listdir for unix-like OSes.  I tested it
under linux.

#!/usr/bin/python
"""
An equivalent os.listdir but as a generator using ctypes
"""

from ctypes import CDLL, c_char_p, c_int, c_long, c_ushort, c_byte, c_char, Structure, POINTER
from ctypes.util import find_library

class c_dir(Structure):
    """Opaque type for directory entries, corresponds to struct DIR"""
c_dir_p = POINTER(c_dir)

class c_dirent(Structure):
    """Directory entry"""
    # FIXME not sure these are the exactly correct types!
    _fields_ = (
        ('d_ino', c_long),            # inode number
        ('d_off', c_long),            # offset to the next dirent
        ('d_reclen', c_ushort),       # length of this record
        ('d_type', c_byte),           # type of file; not supported by all file system types
        ('d_name', c_char * 4096)     # filename
        )
c_dirent_p = POINTER(c_dirent)

c_lib = CDLL(find_library("c"))
opendir = c_lib.opendir
opendir.argtypes = [c_char_p]
opendir.restype = c_dir_p

# FIXME Should probably use readdir_r here
readdir = c_lib.readdir
readdir.argtypes = [c_dir_p]
readdir.restype = c_dirent_p

closedir = c_lib.closedir
closedir.argtypes = [c_dir_p]
closedir.restype = c_int

def listdir(path):
    """
    A generator to return the names of files in the directory passed in
    """
    dir_p = opendir(".")
    while True:
        p = readdir(dir_p)
        if not p:
            break
        name = p.contents.d_name
        if name not in (".", ".."):
            yield name
    closedir(dir_p)

if __name__ == "__main__":
    for name in listdir("."):
        print name

-- 
Nick Craig-Wood <nick at craig-wood.com> -- http://www.craig-wood.com/nick