Jeremy> I ask because some simple httplib tests aren't working as Jeremy> expected with timeouts turned on. I'm also curious because the Jeremy> socket documentation says you shouldn't mix timeouts and Jeremy> makefile(), and httplib uses makefile().

In theory, the change I made to socket.py a couple months ago to always use its _socketobject (and thus _fileobject) class should allow timeouts to be used with makefile. I'd like to know if (and why) you're apparently not getting that code.

Jeremy> When I do have a small timeout and a slow server, I sometimes see an Jeremy> error like this:

...

Jeremy> self.connect() Jeremy> File "/home/jeremy/src/python/dist/src/Lib/httplib.py", line 546, in connect Jeremy> raise socket.error, msg Jeremy> IOError: [Errno socket error] (114, 'Operation already in progress')

Do you perhaps need to cvs up? (Or have you modified your copy of httplib.py?) I'm cvs up'd and line 546 is the "def close" line two lines after the above raise statement.

Skip

When I do have a small timeout and a slow server, I sometimes see an error like this:

[...]

IOError: [Errno socket error] (114, 'Operation already in progress')

Could it be that, while the error isn't perfect, this just means that the socket timed out? After all you did say you had a small timeout and a slow server. Wouldn't you expect it to fail?

Could you try this again with this patch by Bob Halley? It's been proposed as a fix for SF 758239 and while a bit long seems to address the issue by making timeouts a separate exception. I hope that it can go into 2.3b2 tonight.

Index: socketmodule.c =================================================================== RCS file: /cvsroot/python/python/dist/src/Modules/socketmodule.c,v retrieving revision 1.268 diff -c -r1.268 socketmodule.c *** socketmodule.c 10 May 2003 07:36:55 -0000 1.268 --- socketmodule.c 25 Jun 2003 04:39:10 -0000 *************** *** 328,333 **** --- 328,334 ---- static PyObject *socket_error; static PyObject *socket_herror; static PyObject *socket_gaierror; + static PyObject *socket_timeout;

#ifdef RISCOS /* Global variable which is !=0 if Python is running in a RISC OS taskwindow */ *************** *** 575,595 ****

/* Do a select() on the socket, if necessary (sock_timeout > 0). The argument writing indicates the direction. ! This does not raise an exception or return a success indicator; ! we'll let the actual socket call do that. */ ! static void internal_select(PySocketSockObject *s, int writing) { fd_set fds; struct timeval tv;

/* Nothing to do unless we're in timeout mode (not non-blocking) */ if (s->sock_timeout <= 0.0) ! return;

/* Guard against closed socket */ if (s->sock_fd < 0) ! return;

/* Construct the arguments to select */ tv.tv_sec = (int)s->sock_timeout; --- 576,598 ----

/* Do a select() on the socket, if necessary (sock_timeout > 0). The argument writing indicates the direction. ! This does not raise an exception; we'll let our caller do that ! after they've reacquired the interpreter lock. ! Returns 1 on timeout, 0 otherwise. */ ! static int internal_select(PySocketSockObject *s, int writing) { fd_set fds; struct timeval tv; + int n;

/* Nothing to do unless we're in timeout mode (not non-blocking) */ if (s->sock_timeout <= 0.0) ! return 0;

/* Guard against closed socket */ if (s->sock_fd < 0) ! return 0;

/* Construct the arguments to select */ tv.tv_sec = (int)s->sock_timeout; *************** *** 599,607 ****

/* See if the socket is ready */ if (writing) ! select(s->sock_fd+1, NULL, &fds, NULL, &tv); else ! select(s->sock_fd+1, &fds, NULL, NULL, &tv); }

/* Initialize a new socket object. */ --- 602,613 ----

/* See if the socket is ready */ if (writing) ! n = select(s->sock_fd+1, NULL, &fds, NULL, &tv); else ! n = select(s->sock_fd+1, &fds, NULL, NULL, &tv); ! if (n == 0) ! return 1; ! return 0; }

/* Initialize a new socket object. */ *************** *** 1090,1105 **** PyObject *sock = NULL; PyObject *addr = NULL; PyObject *res = NULL;

if (!getsockaddrlen(s, &addrlen)) return NULL; memset(addrbuf, 0, addrlen);

Py_BEGIN_ALLOW_THREADS ! internal_select(s, 0); ! newfd = accept(s->sock_fd, (struct sockaddr *) addrbuf, &addrlen); Py_END_ALLOW_THREADS

#ifdef MS_WINDOWS if (newfd == INVALID_SOCKET) #else --- 1096,1125 ---- PyObject *sock = NULL; PyObject *addr = NULL; PyObject *res = NULL; + int timeout;

if (!getsockaddrlen(s, &addrlen)) return NULL; memset(addrbuf, 0, addrlen);

+ #ifdef MS_WINDOWS + newfd = INVALID_SOCKET; + #else + newfd = -1; + #endif + Py_BEGIN_ALLOW_THREADS ! timeout = internal_select(s, 0); ! if (!timeout) ! newfd = accept(s->sock_fd, (struct sockaddr *) addrbuf, ! &addrlen); Py_END_ALLOW_THREADS

+ if (timeout) { + PyErr_SetString(socket_timeout, "timed out"); + return NULL; + } + #ifdef MS_WINDOWS if (newfd == INVALID_SOCKET) #else *************** *** 1405,1414 **** Close the socket. It cannot be used after this call.");

static int ! internal_connect(PySocketSockObject *s, struct sockaddr *addr, int addrlen) { ! int res;

res = connect(s->sock_fd, addr, addrlen);

#ifdef MS_WINDOWS --- 1425,1436 ---- Close the socket. It cannot be used after this call.");

static int ! internal_connect(PySocketSockObject *s, struct sockaddr *addr, int addrlen, ! int *timeoutp) { ! int res, timeout;

+ timeout = 0; res = connect(s->sock_fd, addr, addrlen);

#ifdef MS_WINDOWS *************** *** 1423,1431 **** FD_ZERO(&fds); FD_SET(s->sock_fd, &fds); res = select(s->sock_fd+1, NULL, &fds, NULL, &tv); ! if (res == 0) res = WSAEWOULDBLOCK; ! else if (res > 0) res = 0; /* else if (res < 0) an error occurred */ } --- 1445,1454 ---- FD_ZERO(&fds); FD_SET(s->sock_fd, &fds); res = select(s->sock_fd+1, NULL, &fds, NULL, &tv); ! if (res == 0) { res = WSAEWOULDBLOCK; ! timeout = 1; ! } else if (res > 0) res = 0; /* else if (res < 0) an error occurred */ } *************** *** 1438,1444 ****

if (s->sock_timeout > 0.0) { if (res < 0 && errno == EINPROGRESS) { ! internal_select(s, 1); res = connect(s->sock_fd, addr, addrlen); if (res < 0 && errno == EISCONN) res = 0; --- 1461,1467 ----

if (s->sock_timeout > 0.0) { if (res < 0 && errno == EINPROGRESS) { ! timeout = internal_select(s, 1); res = connect(s->sock_fd, addr, addrlen); if (res < 0 && errno == EISCONN) res = 0; *************** *** 1449,1454 **** --- 1472,1478 ---- res = errno;

#endif + *timeoutp = timeout;

return res; } *************** *** 1461,1474 **** struct sockaddr *addr; int addrlen; int res;

if (!getsockaddrarg(s, addro, &addr, &addrlen)) return NULL;

Py_BEGIN_ALLOW_THREADS ! res = internal_connect(s, addr, addrlen); Py_END_ALLOW_THREADS

if (res != 0) return s->errorhandler(); Py_INCREF(Py_None); --- 1485,1503 ---- struct sockaddr *addr; int addrlen; int res; + int timeout;

if (!getsockaddrarg(s, addro, &addr, &addrlen)) return NULL;

Py_BEGIN_ALLOW_THREADS ! res = internal_connect(s, addr, addrlen, &timeout); Py_END_ALLOW_THREADS

+ if (timeout) { + PyErr_SetString(socket_timeout, "timed out"); + return NULL; + } if (res != 0) return s->errorhandler(); Py_INCREF(Py_None); *************** *** 1490,1501 **** struct sockaddr *addr; int addrlen; int res;

if (!getsockaddrarg(s, addro, &addr, &addrlen)) return NULL;

Py_BEGIN_ALLOW_THREADS ! res = internal_connect(s, addr, addrlen); Py_END_ALLOW_THREADS

return PyInt_FromLong((long) res); --- 1519,1531 ---- struct sockaddr *addr; int addrlen; int res; + int timeout;

if (!getsockaddrarg(s, addro, &addr, &addrlen)) return NULL;

Py_BEGIN_ALLOW_THREADS ! res = internal_connect(s, addr, addrlen, &timeout); Py_END_ALLOW_THREADS

return PyInt_FromLong((long) res); *************** *** 1716,1722 **** static PyObject * sock_recv(PySocketSockObject *s, PyObject *args) { ! int len, n, flags = 0; PyObject *buf; #ifdef __VMS int read_length; --- 1746,1752 ---- static PyObject * sock_recv(PySocketSockObject *s, PyObject *args) { ! int len, n = 0, flags = 0, timeout; PyObject *buf; #ifdef __VMS int read_length; *************** *** 1738,1747 ****

#ifndef __VMS Py_BEGIN_ALLOW_THREADS ! internal_select(s, 0); ! n = recv(s->sock_fd, PyString_AS_STRING(buf), len, flags); Py_END_ALLOW_THREADS

if (n < 0) { Py_DECREF(buf); return s->errorhandler(); --- 1768,1783 ----

#ifndef __VMS Py_BEGIN_ALLOW_THREADS ! timeout = internal_select(s, 0); ! if (!timeout) ! n = recv(s->sock_fd, PyString_AS_STRING(buf), len, flags); Py_END_ALLOW_THREADS

+ if (timeout) { + Py_DECREF(buf); + PyErr_SetString(socket_timeout, "timed out"); + return NULL; + } if (n < 0) { Py_DECREF(buf); return s->errorhandler(); *************** *** 1763,1772 **** }

Py_BEGIN_ALLOW_THREADS ! internal_select(s, 0); ! n = recv(s->sock_fd, read_buf, segment, flags); Py_END_ALLOW_THREADS

if (n < 0) { Py_DECREF(buf); return s->errorhandler(); --- 1799,1814 ---- }

Py_BEGIN_ALLOW_THREADS ! timeout = internal_select(s, 0); ! if (!timeout) ! n = recv(s->sock_fd, read_buf, segment, flags); Py_END_ALLOW_THREADS

+ if (timeout) { + Py_DECREF(buf); + PyErr_SetString(socket_timeout, "timed out"); + return NULL; + } if (n < 0) { Py_DECREF(buf); return s->errorhandler(); *************** *** 1805,1811 **** PyObject *buf = NULL; PyObject *addr = NULL; PyObject *ret = NULL; ! int len, n, flags = 0; socklen_t addrlen;

if (!PyArg_ParseTuple(args, "i|i:recvfrom", &len, &flags)) --- 1847,1853 ---- PyObject *buf = NULL; PyObject *addr = NULL; PyObject *ret = NULL; ! int len, n = 0, flags = 0, timeout; socklen_t addrlen;

if (!PyArg_ParseTuple(args, "i|i:recvfrom", &len, &flags)) *************** *** 1819,1838 ****

Py_BEGIN_ALLOW_THREADS memset(addrbuf, 0, addrlen); ! internal_select(s, 0); ! n = recvfrom(s->sock_fd, PyString_AS_STRING(buf), len, flags, #ifndef MS_WINDOWS #if defined(PYOS_OS2) && !defined(PYCC_GCC) ! (struct sockaddr *)addrbuf, &addrlen #else ! (void *)addrbuf, &addrlen #endif #else ! (struct sockaddr *)addrbuf, &addrlen #endif ! ); Py_END_ALLOW_THREADS

if (n < 0) { Py_DECREF(buf); return s->errorhandler(); --- 1861,1886 ----

Py_BEGIN_ALLOW_THREADS memset(addrbuf, 0, addrlen); ! timeout = internal_select(s, 0); ! if (!timeout) ! n = recvfrom(s->sock_fd, PyString_AS_STRING(buf), len, flags, #ifndef MS_WINDOWS #if defined(PYOS_OS2) && !defined(PYCC_GCC) ! (struct sockaddr *)addrbuf, &addrlen #else ! (void *)addrbuf, &addrlen #endif #else ! (struct sockaddr *)addrbuf, &addrlen #endif ! ); Py_END_ALLOW_THREADS

+ if (timeout) { + Py_DECREF(buf); + PyErr_SetString(socket_timeout, "timed out"); + return NULL; + } if (n < 0) { Py_DECREF(buf); return s->errorhandler(); *************** *** 1864,1870 **** sock_send(PySocketSockObject *s, PyObject *args) { char *buf; ! int len, n, flags = 0; #ifdef __VMS int send_length; #endif --- 1912,1918 ---- sock_send(PySocketSockObject *s, PyObject *args) { char *buf; ! int len, n = 0, flags = 0, timeout; #ifdef __VMS int send_length; #endif *************** *** 1874,1883 ****

#ifndef __VMS Py_BEGIN_ALLOW_THREADS ! internal_select(s, 1); ! n = send(s->sock_fd, buf, len, flags); Py_END_ALLOW_THREADS

if (n < 0) return s->errorhandler(); #else --- 1922,1936 ----

#ifndef __VMS Py_BEGIN_ALLOW_THREADS ! timeout = internal_select(s, 1); ! if (!timeout) ! n = send(s->sock_fd, buf, len, flags); Py_END_ALLOW_THREADS

+ if (timeout) { + PyErr_SetString(socket_timeout, "timed out"); + return NULL; + } if (n < 0) return s->errorhandler(); #else *************** *** 1895,1903 **** segment = send_length; } Py_BEGIN_ALLOW_THREADS ! internal_select(s, 1); ! n = send(s->sock_fd, buf, segment, flags); Py_END_ALLOW_THREADS if (n < 0) { return s->errorhandler(); } --- 1948,1961 ---- segment = send_length; } Py_BEGIN_ALLOW_THREADS ! timeout = internal_select(s, 1); ! if (!timeout) ! n = send(s->sock_fd, buf, segment, flags); Py_END_ALLOW_THREADS + if (timeout) { + PyErr_SetString(socket_timeout, "timed out"); + return NULL; + } if (n < 0) { return s->errorhandler(); } *************** *** 1922,1935 **** sock_sendall(PySocketSockObject *s, PyObject *args) { char *buf; ! int len, n, flags = 0;

if (!PyArg_ParseTuple(args, "s#|i:sendall", &buf, &len, &flags)) return NULL;

Py_BEGIN_ALLOW_THREADS do { ! internal_select(s, 1); n = send(s->sock_fd, buf, len, flags); if (n < 0) break; --- 1980,1995 ---- sock_sendall(PySocketSockObject *s, PyObject *args) { char *buf; ! int len, n = 0, flags = 0, timeout;

if (!PyArg_ParseTuple(args, "s#|i:sendall", &buf, &len, &flags)) return NULL;

Py_BEGIN_ALLOW_THREADS do { ! timeout = internal_select(s, 1); ! if (timeout) ! break; n = send(s->sock_fd, buf, len, flags); if (n < 0) break; *************** *** 1938,1943 **** --- 1998,2007 ---- } while (len > 0); Py_END_ALLOW_THREADS

+ if (timeout) { + PyErr_SetString(socket_timeout, "timed out"); + return NULL; + } if (n < 0) return s->errorhandler();

*************** *** 1962,1968 **** PyObject *addro; char *buf; struct sockaddr *addr; ! int addrlen, len, n, flags;

flags = 0; if (!PyArg_ParseTuple(args, "s#O:sendto", &buf, &len, &addro)) { --- 2026,2032 ---- PyObject *addro; char *buf; struct sockaddr *addr; ! int addrlen, len, n = 0, flags, timeout;

flags = 0; if (!PyArg_ParseTuple(args, "s#O:sendto", &buf, &len, &addro)) { *************** *** 1976,1985 **** return NULL;

Py_BEGIN_ALLOW_THREADS ! internal_select(s, 1); ! n = sendto(s->sock_fd, buf, len, flags, addr, addrlen); Py_END_ALLOW_THREADS

if (n < 0) return s->errorhandler(); return PyInt_FromLong((long)n); --- 2040,2054 ---- return NULL;

Py_BEGIN_ALLOW_THREADS ! timeout = internal_select(s, 1); ! if (!timeout) ! n = sendto(s->sock_fd, buf, len, flags, addr, addrlen); Py_END_ALLOW_THREADS

+ if (timeout) { + PyErr_SetString(socket_timeout, "timed out"); + return NULL; + } if (n < 0) return s->errorhandler(); return PyInt_FromLong((long)n); *************** *** 3409,3414 **** --- 3478,3489 ---- return; Py_INCREF(socket_gaierror); PyModule_AddObject(m, "gaierror", socket_gaierror); + socket_timeout = PyErr_NewException("socket.timeout", + socket_error, NULL); + if (socket_timeout == NULL) + return; + Py_INCREF(socket_timeout); + PyModule_AddObject(m, "timeout", socket_timeout); Py_INCREF((PyObject *)&sock_type); if (PyModule_AddObject(m, "SocketType", (PyObject *)&sock_type) != 0)

--Guido van Rossum (home page: http://www.python.org/~guido/)