waling a directory with very many files

[Nick Craig-Wood]

Nick Craig-Wood <nick at craig-wood.com> wrote:
>  Jean-Paul Calderone <exarkun at divmod.com> wrote:
> >  On Mon, 15 Jun 2009 09:29:33 -0500, Nick Craig-Wood <nick at craig-wood.com> wrote:
> > >Hrvoje Niksic <hniksic at xemacs.org> wrote:
> > >>  Nick Craig-Wood <nick at craig-wood.com> writes:
> > >>
> > >> > Here is a ctypes generator listdir for unix-like OSes.
> > >>
> > >>  ctypes code scares me with its duplication of the contents of system
> > >>  headers.  I understand its use as a proof of concept, or for hacks one
> > >>  needs right now, but can anyone seriously propose using this kind of
> > >>  code in a Python program?  For example, this seems much more
> > >>  "Linux-only", or possibly even "32-bit-Linux-only", than
> > >>  "unix-like":
> > >
> > >It was a proof of concept certainly..

Just in case anyone is interested here is an implementation using cython.

Compile with python setup.py build_ext --inplace

And run listdir.py

This would have been much easier if cython supported yield, but
unfortunately it doesn't (yet - I think it is in the works).

This really should work on any platform!

--setup.py----------------------------------------------------------
from distutils.core import setup
from distutils.extension import Extension
from Cython.Distutils import build_ext

setup(
    cmdclass = {'build_ext': build_ext},
    ext_modules = [Extension("directory", ["directory.pyx"])]
)

--directory.pyx----------------------------------------------------------
# Cython interface for listdir
# python setup.py build_ext --inplace

import cython

cdef extern from "dirent.h":
    struct dirent:
        char d_name[0]
    struct dir_handle:
        pass
    ctypedef dir_handle DIR "DIR"
    DIR *opendir(char *name)
    int closedir(DIR *dirp)
    dirent *readdir(DIR *dirp)

cdef class Directory:
    """Represents an open directory"""

    cdef DIR *handle

    def __init__(self, path):
        self.handle = opendir(path)

    def readdir(self):
        cdef dirent *p
        p = readdir(self.handle)
        if p is NULL:
            return None
        return p.d_name

    def close(self):
        closedir(self.handle)

--listdir.py----------------------------------------------------------
from directory import Directory

def listdir(path):
    """
    A generator to return the names of files in the directory passed
    in
    """
    d = Directory(".")
    while True:
        name = d.readdir()
        if not name:
            break
        if name not in (".", ".."):
            yield name
    d.close()

if __name__ == "__main__":
    for name in listdir("."):
        print name
------------------------------------------------------------


-- 
Nick Craig-Wood <nick at craig-wood.com> -- http://www.craig-wood.com/nick