select() vs. pipes (was [Python-Dev] Re: PEP 324 (process module))

[Chris McDonough]

On Wed, 2004-08-04 at 16:03, Chris McDonough wrote:
> I was all set to try to refute this, but after writing a minimal test
> program to do what I want to do, I find that you're right.  That's good
> news!  I'll need to revisit my workaroudns in the program that caused me
> to need to do this.  Thanks for the schooling.

Ugh.  I spoke a bit too soon..

The following program demonstrates that a particular usage of select
(under Linux at least) always returns the output side of a pipe
connected to a child process' stdout as "ready" after it gets any output
from that child process, even if the child process has no further data
to provide after it has provided a bit of data to the parent.  This is
what causes the "busywait" behavior I've experienced in the past (note
that as this program runs, your CPU utilization will likely be near
100%).

Or am I doing something silly?

import select
import errno
import fcntl
import os
import stat
import sys
from fcntl import F_SETFL, F_GETFL

def get_path():
    """Return a list corresponding to $PATH, or a default."""
    path = ["/bin", "/usr/bin", "/usr/local/bin"]
    if os.environ.has_key("PATH"):
        p = os.environ["PATH"]
        if p:
            path = p.split(os.pathsep)
    return path

def get_execv_args(command):
    """Internal: turn a program name into a file name, using 
    $PATH."""
    commandargs = command.split()
    program = commandargs[0]
    if "/" in program:
        filename = program
        try:
            st = os.stat(filename)
        except os.error:
            return None, None
    else:
        path = get_path()
        for dir in path:
            filename = os.path.join(dir, program)
            try:
                st = os.stat(filename)
            except os.error:
                continue
            mode = st[stat.ST_MODE]
            if mode & 0111:
                break
        else:
            return None, None
    if not os.access(filename, os.X_OK):
        return None, None
    return filename, commandargs

def spawn(command):
    """Start the subprocess."""

    filename, argv = get_execv_args(command)
    if filename is None:
        raise RuntimeError, '%s is an invalid command' % command

    child_stdin, stdin = os.pipe()
    stdout, child_stdout = os.pipe()
    stderr, child_stderr = os.pipe()

    # open stderr, stdout in nonblocking mode so we can tail them
    # in the mainloop without blocking
    for fd in stdout, stderr:
        flags = fcntl.fcntl(fd, F_GETFL)
        fcntl.fcntl(fd, F_SETFL, flags | os.O_NDELAY)

    pid = os.fork()

    if pid != 0:
        # Parent
        os.close(child_stdin)
        os.close(child_stdout)
        os.close(child_stderr)
        return stdin, stdout, stderr

    else:
        # Child
        try:
            os.dup2(child_stdin, 0)
            os.dup2(child_stdout, 1)
            os.dup2(child_stderr, 2)
            for i in range(3, 256):
                try:
                    os.close(i)
                except:
                    pass
            os.execv(filename, argv)
        finally:
            os._exit(127)

def go(out_fds):
    while 1:
        try:
            r, w, x = select.select(out_fds, [], [], 1)
            if not r:
                print "timed out"
        except select.error, err:
            if err[0] != errno.EINTR:
                raise
        for fd in r:
            sys.stdout.write(os.read(fd, 1024))

stdin, stderr, stdout = spawn('echo "foo"')

go([stderr, stdout])