[Python-Dev] socket timeouts and httplib

Guido van Rossum guido@python.org
Sat, 28 Jun 2003 17:51:26 -0400


> When I do have a small timeout and a slow server, I sometimes see an
> error like this:
[...]
> IOError: [Errno socket error] (114, 'Operation already in progress')

Could it be that, while the error isn't perfect, this just means that
the socket timed out?  After all you did say you had a small timeout
and a slow server.  Wouldn't you expect it to fail?

Could you try this again with this patch by Bob Halley?  It's been
proposed as a fix for SF 758239 and while a bit long seems to address
the issue by making timeouts a separate exception.  I hope that it can
go into 2.3b2 tonight.

Index: socketmodule.c
===================================================================
RCS file: /cvsroot/python/python/dist/src/Modules/socketmodule.c,v
retrieving revision 1.268
diff -c -r1.268 socketmodule.c
*** socketmodule.c	10 May 2003 07:36:55 -0000	1.268
--- socketmodule.c	25 Jun 2003 04:39:10 -0000
***************
*** 328,333 ****
--- 328,334 ----
  static PyObject *socket_error;
  static PyObject *socket_herror;
  static PyObject *socket_gaierror;
+ static PyObject *socket_timeout;
  
  #ifdef RISCOS
  /* Global variable which is !=0 if Python is running in a RISC OS taskwindow */
***************
*** 575,595 ****
  
  /* Do a select() on the socket, if necessary (sock_timeout > 0).
     The argument writing indicates the direction.
!    This does not raise an exception or return a success indicator;
!    we'll let the actual socket call do that. */
! static void
  internal_select(PySocketSockObject *s, int writing)
  {
  	fd_set fds;
  	struct timeval tv;
  
  	/* Nothing to do unless we're in timeout mode (not non-blocking) */
  	if (s->sock_timeout <= 0.0)
! 		return;
  
  	/* Guard against closed socket */
  	if (s->sock_fd < 0)
! 		return;
  
  	/* Construct the arguments to select */
  	tv.tv_sec = (int)s->sock_timeout;
--- 576,598 ----
  
  /* Do a select() on the socket, if necessary (sock_timeout > 0).
     The argument writing indicates the direction.
!    This does not raise an exception; we'll let our caller do that
!    after they've reacquired the interpreter lock.
!    Returns 1 on timeout, 0 otherwise. */
! static int
  internal_select(PySocketSockObject *s, int writing)
  {
  	fd_set fds;
  	struct timeval tv;
+ 	int n;
  
  	/* Nothing to do unless we're in timeout mode (not non-blocking) */
  	if (s->sock_timeout <= 0.0)
! 		return 0;
  
  	/* Guard against closed socket */
  	if (s->sock_fd < 0)
! 		return 0;
  
  	/* Construct the arguments to select */
  	tv.tv_sec = (int)s->sock_timeout;
***************
*** 599,607 ****
  
  	/* See if the socket is ready */
  	if (writing)
! 		select(s->sock_fd+1, NULL, &fds, NULL, &tv);
  	else
! 		select(s->sock_fd+1, &fds, NULL, NULL, &tv);
  }
  
  /* Initialize a new socket object. */
--- 602,613 ----
  
  	/* See if the socket is ready */
  	if (writing)
! 		n = select(s->sock_fd+1, NULL, &fds, NULL, &tv);
  	else
! 		n = select(s->sock_fd+1, &fds, NULL, NULL, &tv);
! 	if (n == 0)
! 		return 1;
! 	return 0;
  }
  
  /* Initialize a new socket object. */
***************
*** 1090,1105 ****
  	PyObject *sock = NULL;
  	PyObject *addr = NULL;
  	PyObject *res = NULL;
  
  	if (!getsockaddrlen(s, &addrlen))
  		return NULL;
  	memset(addrbuf, 0, addrlen);
  
  	Py_BEGIN_ALLOW_THREADS
! 	internal_select(s, 0);
! 	newfd = accept(s->sock_fd, (struct sockaddr *) addrbuf, &addrlen);
  	Py_END_ALLOW_THREADS
  
  #ifdef MS_WINDOWS
  	if (newfd == INVALID_SOCKET)
  #else
--- 1096,1125 ----
  	PyObject *sock = NULL;
  	PyObject *addr = NULL;
  	PyObject *res = NULL;
+ 	int timeout;
  
  	if (!getsockaddrlen(s, &addrlen))
  		return NULL;
  	memset(addrbuf, 0, addrlen);
  
+ #ifdef MS_WINDOWS
+ 	newfd = INVALID_SOCKET;
+ #else
+ 	newfd = -1;
+ #endif
+ 
  	Py_BEGIN_ALLOW_THREADS
! 	timeout = internal_select(s, 0);
! 	if (!timeout)
! 		newfd = accept(s->sock_fd, (struct sockaddr *) addrbuf,
! 			       &addrlen);
  	Py_END_ALLOW_THREADS
  
+ 	if (timeout) {
+ 		PyErr_SetString(socket_timeout, "timed out");
+ 		return NULL;
+ 	}
+ 
  #ifdef MS_WINDOWS
  	if (newfd == INVALID_SOCKET)
  #else
***************
*** 1405,1414 ****
  Close the socket.  It cannot be used after this call.");
  
  static int
! internal_connect(PySocketSockObject *s, struct sockaddr *addr, int addrlen)
  {
! 	int res;
  
  	res = connect(s->sock_fd, addr, addrlen);
  
  #ifdef MS_WINDOWS
--- 1425,1436 ----
  Close the socket.  It cannot be used after this call.");
  
  static int
! internal_connect(PySocketSockObject *s, struct sockaddr *addr, int addrlen,
! 		 int *timeoutp)
  {
! 	int res, timeout;
  
+ 	timeout = 0;
  	res = connect(s->sock_fd, addr, addrlen);
  
  #ifdef MS_WINDOWS
***************
*** 1423,1431 ****
  			FD_ZERO(&fds);
  			FD_SET(s->sock_fd, &fds);
  			res = select(s->sock_fd+1, NULL, &fds, NULL, &tv);
! 			if (res == 0)
  				res = WSAEWOULDBLOCK;
! 			else if (res > 0)
  				res = 0;
  			/* else if (res < 0) an error occurred */
  		}
--- 1445,1454 ----
  			FD_ZERO(&fds);
  			FD_SET(s->sock_fd, &fds);
  			res = select(s->sock_fd+1, NULL, &fds, NULL, &tv);
! 			if (res == 0) {
  				res = WSAEWOULDBLOCK;
! 				timeout = 1;
! 			} else if (res > 0)
  				res = 0;
  			/* else if (res < 0) an error occurred */
  		}
***************
*** 1438,1444 ****
  
  	if (s->sock_timeout > 0.0) {
  		if (res < 0 && errno == EINPROGRESS) {
! 			internal_select(s, 1);
  			res = connect(s->sock_fd, addr, addrlen);
  			if (res < 0 && errno == EISCONN)
  				res = 0;
--- 1461,1467 ----
  
  	if (s->sock_timeout > 0.0) {
  		if (res < 0 && errno == EINPROGRESS) {
! 			timeout = internal_select(s, 1);
  			res = connect(s->sock_fd, addr, addrlen);
  			if (res < 0 && errno == EISCONN)
  				res = 0;
***************
*** 1449,1454 ****
--- 1472,1478 ----
  		res = errno;
  
  #endif
+ 	*timeoutp = timeout;
  
  	return res;
  }
***************
*** 1461,1474 ****
  	struct sockaddr *addr;
  	int addrlen;
  	int res;
  
  	if (!getsockaddrarg(s, addro, &addr, &addrlen))
  		return NULL;
  
  	Py_BEGIN_ALLOW_THREADS
! 	res = internal_connect(s, addr, addrlen);
  	Py_END_ALLOW_THREADS
  
  	if (res != 0)
  		return s->errorhandler();
  	Py_INCREF(Py_None);
--- 1485,1503 ----
  	struct sockaddr *addr;
  	int addrlen;
  	int res;
+ 	int timeout;
  
  	if (!getsockaddrarg(s, addro, &addr, &addrlen))
  		return NULL;
  
  	Py_BEGIN_ALLOW_THREADS
! 	res = internal_connect(s, addr, addrlen, &timeout);
  	Py_END_ALLOW_THREADS
  
+ 	if (timeout) {
+ 		PyErr_SetString(socket_timeout, "timed out");
+ 		return NULL;
+ 	}
  	if (res != 0)
  		return s->errorhandler();
  	Py_INCREF(Py_None);
***************
*** 1490,1501 ****
  	struct sockaddr *addr;
  	int addrlen;
  	int res;
  
  	if (!getsockaddrarg(s, addro, &addr, &addrlen))
  		return NULL;
  
  	Py_BEGIN_ALLOW_THREADS
! 	res = internal_connect(s, addr, addrlen);
  	Py_END_ALLOW_THREADS
  
  	return PyInt_FromLong((long) res);
--- 1519,1531 ----
  	struct sockaddr *addr;
  	int addrlen;
  	int res;
+ 	int timeout;
  
  	if (!getsockaddrarg(s, addro, &addr, &addrlen))
  		return NULL;
  
  	Py_BEGIN_ALLOW_THREADS
! 	res = internal_connect(s, addr, addrlen, &timeout);
  	Py_END_ALLOW_THREADS
  
  	return PyInt_FromLong((long) res);
***************
*** 1716,1722 ****
  static PyObject *
  sock_recv(PySocketSockObject *s, PyObject *args)
  {
! 	int len, n, flags = 0;
  	PyObject *buf;
  #ifdef __VMS
  	int read_length;
--- 1746,1752 ----
  static PyObject *
  sock_recv(PySocketSockObject *s, PyObject *args)
  {
! 	int len, n = 0, flags = 0, timeout;
  	PyObject *buf;
  #ifdef __VMS
  	int read_length;
***************
*** 1738,1747 ****
  
  #ifndef __VMS
  	Py_BEGIN_ALLOW_THREADS
! 	internal_select(s, 0);
! 	n = recv(s->sock_fd, PyString_AS_STRING(buf), len, flags);
  	Py_END_ALLOW_THREADS
  
  	if (n < 0) {
  		Py_DECREF(buf);
  		return s->errorhandler();
--- 1768,1783 ----
  
  #ifndef __VMS
  	Py_BEGIN_ALLOW_THREADS
! 	timeout = internal_select(s, 0);
! 	if (!timeout)
! 		n = recv(s->sock_fd, PyString_AS_STRING(buf), len, flags);
  	Py_END_ALLOW_THREADS
  
+ 	if (timeout) {
+ 		Py_DECREF(buf);
+ 		PyErr_SetString(socket_timeout, "timed out");
+ 		return NULL;
+ 	}
  	if (n < 0) {
  		Py_DECREF(buf);
  		return s->errorhandler();
***************
*** 1763,1772 ****
  		}
  
  		Py_BEGIN_ALLOW_THREADS
!  	  	internal_select(s, 0);
! 		n = recv(s->sock_fd, read_buf, segment, flags);
  		Py_END_ALLOW_THREADS
  
  		if (n < 0) {
  			Py_DECREF(buf);
  			return s->errorhandler();
--- 1799,1814 ----
  		}
  
  		Py_BEGIN_ALLOW_THREADS
! 		timeout = internal_select(s, 0);
! 		if (!timeout)
! 			n = recv(s->sock_fd, read_buf, segment, flags);
  		Py_END_ALLOW_THREADS
  
+ 		if (timeout) {
+ 			Py_DECREF(buf);
+ 			PyErr_SetString(socket_timeout, "timed out");
+ 			return NULL;
+ 		}
  		if (n < 0) {
  			Py_DECREF(buf);
  			return s->errorhandler();
***************
*** 1805,1811 ****
  	PyObject *buf = NULL;
  	PyObject *addr = NULL;
  	PyObject *ret = NULL;
! 	int len, n, flags = 0;
  	socklen_t addrlen;
  
  	if (!PyArg_ParseTuple(args, "i|i:recvfrom", &len, &flags))
--- 1847,1853 ----
  	PyObject *buf = NULL;
  	PyObject *addr = NULL;
  	PyObject *ret = NULL;
! 	int len, n = 0, flags = 0, timeout;
  	socklen_t addrlen;
  
  	if (!PyArg_ParseTuple(args, "i|i:recvfrom", &len, &flags))
***************
*** 1819,1838 ****
  
  	Py_BEGIN_ALLOW_THREADS
  	memset(addrbuf, 0, addrlen);
! 	internal_select(s, 0);
! 	n = recvfrom(s->sock_fd, PyString_AS_STRING(buf), len, flags,
  #ifndef MS_WINDOWS
  #if defined(PYOS_OS2) && !defined(PYCC_GCC)
! 		     (struct sockaddr *)addrbuf, &addrlen
  #else
! 		     (void *)addrbuf, &addrlen
  #endif
  #else
! 		     (struct sockaddr *)addrbuf, &addrlen
  #endif
! 		     );
  	Py_END_ALLOW_THREADS
  
  	if (n < 0) {
  		Py_DECREF(buf);
  		return s->errorhandler();
--- 1861,1886 ----
  
  	Py_BEGIN_ALLOW_THREADS
  	memset(addrbuf, 0, addrlen);
! 	timeout = internal_select(s, 0);
! 	if (!timeout)
! 		n = recvfrom(s->sock_fd, PyString_AS_STRING(buf), len, flags,
  #ifndef MS_WINDOWS
  #if defined(PYOS_OS2) && !defined(PYCC_GCC)
! 			     (struct sockaddr *)addrbuf, &addrlen
  #else
! 			     (void *)addrbuf, &addrlen
  #endif
  #else
! 			     (struct sockaddr *)addrbuf, &addrlen
  #endif
! 			);
  	Py_END_ALLOW_THREADS
  
+ 	if (timeout) {
+ 		Py_DECREF(buf);
+ 		PyErr_SetString(socket_timeout, "timed out");
+ 		return NULL;
+ 	}
  	if (n < 0) {
  		Py_DECREF(buf);
  		return s->errorhandler();
***************
*** 1864,1870 ****
  sock_send(PySocketSockObject *s, PyObject *args)
  {
  	char *buf;
! 	int len, n, flags = 0;
  #ifdef __VMS
  	int send_length;
  #endif
--- 1912,1918 ----
  sock_send(PySocketSockObject *s, PyObject *args)
  {
  	char *buf;
! 	int len, n = 0, flags = 0, timeout;
  #ifdef __VMS
  	int send_length;
  #endif
***************
*** 1874,1883 ****
  
  #ifndef __VMS
  	Py_BEGIN_ALLOW_THREADS
! 	internal_select(s, 1);
! 	n = send(s->sock_fd, buf, len, flags);
  	Py_END_ALLOW_THREADS
  
  	if (n < 0)
  		return s->errorhandler();
  #else
--- 1922,1936 ----
  
  #ifndef __VMS
  	Py_BEGIN_ALLOW_THREADS
! 	timeout = internal_select(s, 1);
! 	if (!timeout)
! 		n = send(s->sock_fd, buf, len, flags);
  	Py_END_ALLOW_THREADS
  
+ 	if (timeout) {
+ 		PyErr_SetString(socket_timeout, "timed out");
+ 		return NULL;
+ 	}
  	if (n < 0)
  		return s->errorhandler();
  #else
***************
*** 1895,1903 ****
  			segment = send_length;
  		}
  		Py_BEGIN_ALLOW_THREADS
! 		internal_select(s, 1);
! 		n = send(s->sock_fd, buf, segment, flags);
  		Py_END_ALLOW_THREADS
  		if (n < 0) {
  			return s->errorhandler();
  		}
--- 1948,1961 ----
  			segment = send_length;
  		}
  		Py_BEGIN_ALLOW_THREADS
! 		timeout = internal_select(s, 1);
! 		if (!timeout)
! 			n = send(s->sock_fd, buf, segment, flags);
  		Py_END_ALLOW_THREADS
+ 		if (timeout) {
+ 			PyErr_SetString(socket_timeout, "timed out");
+ 			return NULL;
+ 		}
  		if (n < 0) {
  			return s->errorhandler();
  		}
***************
*** 1922,1935 ****
  sock_sendall(PySocketSockObject *s, PyObject *args)
  {
  	char *buf;
! 	int len, n, flags = 0;
  
  	if (!PyArg_ParseTuple(args, "s#|i:sendall", &buf, &len, &flags))
  		return NULL;
  
  	Py_BEGIN_ALLOW_THREADS
  	do {
! 		internal_select(s, 1);
  		n = send(s->sock_fd, buf, len, flags);
  		if (n < 0)
  			break;
--- 1980,1995 ----
  sock_sendall(PySocketSockObject *s, PyObject *args)
  {
  	char *buf;
! 	int len, n = 0, flags = 0, timeout;
  
  	if (!PyArg_ParseTuple(args, "s#|i:sendall", &buf, &len, &flags))
  		return NULL;
  
  	Py_BEGIN_ALLOW_THREADS
  	do {
! 		timeout = internal_select(s, 1);
! 		if (timeout)
! 			break;
  		n = send(s->sock_fd, buf, len, flags);
  		if (n < 0)
  			break;
***************
*** 1938,1943 ****
--- 1998,2007 ----
  	} while (len > 0);
  	Py_END_ALLOW_THREADS
  
+ 	if (timeout) {
+ 		PyErr_SetString(socket_timeout, "timed out");
+ 		return NULL;
+ 	}
  	if (n < 0)
  		return s->errorhandler();
  
***************
*** 1962,1968 ****
  	PyObject *addro;
  	char *buf;
  	struct sockaddr *addr;
! 	int addrlen, len, n, flags;
  
  	flags = 0;
  	if (!PyArg_ParseTuple(args, "s#O:sendto", &buf, &len, &addro)) {
--- 2026,2032 ----
  	PyObject *addro;
  	char *buf;
  	struct sockaddr *addr;
! 	int addrlen, len, n = 0, flags, timeout;
  
  	flags = 0;
  	if (!PyArg_ParseTuple(args, "s#O:sendto", &buf, &len, &addro)) {
***************
*** 1976,1985 ****
  		return NULL;
  
  	Py_BEGIN_ALLOW_THREADS
! 	internal_select(s, 1);
! 	n = sendto(s->sock_fd, buf, len, flags, addr, addrlen);
  	Py_END_ALLOW_THREADS
  
  	if (n < 0)
  		return s->errorhandler();
  	return PyInt_FromLong((long)n);
--- 2040,2054 ----
  		return NULL;
  
  	Py_BEGIN_ALLOW_THREADS
! 	timeout = internal_select(s, 1);
! 	if (!timeout)
! 		n = sendto(s->sock_fd, buf, len, flags, addr, addrlen);
  	Py_END_ALLOW_THREADS
  
+ 	if (timeout) {
+ 		PyErr_SetString(socket_timeout, "timed out");
+ 		return NULL;
+ 	}
  	if (n < 0)
  		return s->errorhandler();
  	return PyInt_FromLong((long)n);
***************
*** 3409,3414 ****
--- 3478,3489 ----
  		return;
  	Py_INCREF(socket_gaierror);
  	PyModule_AddObject(m, "gaierror", socket_gaierror);
+ 	socket_timeout = PyErr_NewException("socket.timeout",
+ 					    socket_error, NULL);
+ 	if (socket_timeout == NULL)
+ 		return;
+ 	Py_INCREF(socket_timeout);
+ 	PyModule_AddObject(m, "timeout", socket_timeout);
  	Py_INCREF((PyObject *)&sock_type);
  	if (PyModule_AddObject(m, "SocketType",
  			       (PyObject *)&sock_type) != 0)



--Guido van Rossum (home page: http://www.python.org/~guido/)