[Python-Dev] HTTP/1.1 capable httplib module

Jeremy Hylton jeremy@alum.mit.edu
Fri, 19 May 2000 17:46:11 -0700 (PDT)

I applied the recent changes to the CVS httplib to Greg's httplib
(call it httplib11) this afternoon.  The result is included below.  I
think this is quite close to checking in, but it could use a slightly
better test suite.

There are a few outstanding questions.

httplib11 does not implement the debuglevel feature.  I don't think
it's important, but it is currently documented and may be used.
Guido, should we implement it?

httplib w/SSL uses a constructor with this prototype:
    def __init__(self, host='', port=None, **x509):
It looks like the x509 dictionary should contain two variables --
key_file and cert_file.  Since we know what the entries are, why not
make them explicit?
    def __init__(self, host='', port=None, cert_file=None, key_file=None):
(Or reverse the two arguments if that is clearer.)

The FakeSocket class in CVS has a comment after the makefile def line
that says "hopefully, never have to write."  It won't do at all the
right thing when called with a write mode, so it ought to raise an
exception.  Any reason it doesn't?

I'd like to add a couple of test cases that use HTTP/1.1 to get some
pages from python.org, including one that uses the chunked encoding.
Just haven't gotten around to it.  Question on that front: Does it
make sense to incorporate the test function in the module with the std
regression test suite?  In general, I would think so.  In this
particular case, the test could fail because of host networking
problems.  I think that's okay as long as the error message is clear


"""HTTP/1.1 client library"""

# Written by Greg Stein.

import socket
import string
import mimetools

    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

error = 'httplib.error'


class HTTPResponse(mimetools.Message):
    __super_init = mimetools.Message.__init__
    def __init__(self, fp, version, errcode):
        self.__super_init(fp, 0)

        if version == 'HTTP/1.0':
            self.version = 10
        elif version[:7] == 'HTTP/1.':
            self.version = 11 # use HTTP/1.1 code for HTTP/1.x where x>=1
            raise error, 'unknown HTTP protocol'

        # are we using the chunked-style of transfer encoding?
        tr_enc = self.getheader('transfer-encoding')
        if tr_enc:
            if string.lower(tr_enc) != 'chunked':
                raise error, 'unknown transfer-encoding'
            self.chunked = 1
            self.chunk_left = None
            self.chunked = 0

        # will the connection close at the end of the response?
        conn = self.getheader('connection')
        if conn:
            conn = string.lower(conn)
            # a "Connection: close" will always close the
            # connection. if we don't see that and this is not
            # HTTP/1.1, then the connection will close unless we see a
            # Keep-Alive header. 
            self.will_close = string.find(conn, 'close') != -1 or \
                              ( self.version != 11 and \
                                not self.getheader('keep-alive') )
            # for HTTP/1.1, the connection will always remain open
            # otherwise, it will remain open IFF we see a Keep-Alive header
            self.will_close = self.version != 11 and \
                              not self.getheader('keep-alive')

        # do we have a Content-Length?
        # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"
        length = self.getheader('content-length')
        if length and not self.chunked:
            self.length = int(length)
            self.length = None

        # does the body have a fixed length? (of zero)
        if (errcode == 204 or               # No Content
            errcode == 304 or               # Not Modified
            100 <= errcode < 200):          # 1xx codes
            self.length = 0

        # if the connection remains open, and we aren't using chunked, and
        # a content-length was not provided, then assume that the connection
        # WILL close.
        if not self.will_close and \
           not self.chunked and \
           self.length is None:
            self.will_close = 1

        # if there is no body, then close NOW. read() may never be
        # called, thus we will never mark self as closed.
        if self.length == 0:

    def close(self):
        if self.fp:
            self.fp = None

    def isclosed(self):
        # NOTE: it is possible that we will not ever call self.close(). This
        #       case occurs when will_close is TRUE, length is None, and we
        #       read up to the last byte, but NOT past it.
        # IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be
        #          called, meaning self.isclosed() is meaningful.
        return self.fp is None

    def read(self, amt=None):
        if self.fp is None:
            return ''

        if self.chunked:
            chunk_left = self.chunk_left
            value = ''
            while 1:
                if chunk_left is None:
                    line = self.fp.readline()
                    i = string.find(line, ';')
                    if i >= 0:
                        line = line[:i]     # strip chunk-extensions
                    chunk_left = string.atoi(line, 16)
                    if chunk_left == 0:
                if amt is None:
                    value = value + self.fp.read(chunk_left)
                elif amt < chunk_left:
                    value = value + self.fp.read(amt)
                    self.chunk_left = chunk_left - amt
                    return value
                elif amt == chunk_left:
                    value = value + self.fp.read(amt)
                    self.fp.read(2)    # toss the CRLF at the end of the chunk
                    self.chunk_left = None
                    return value
                    value = value + self.fp.read(chunk_left)
                    amt = amt - chunk_left

                # we read the whole chunk, get another
                self.fp.read(2)        # toss the CRLF at the end of the chunk
                chunk_left = None

            # read and discard trailer up to the CRLF terminator
            ### note: we shouldn't have any trailers!
            while 1:
                line = self.fp.readline()
                if line == '\r\n':

            # we read everything; close the "file"

            return value

        elif amt is None:
            # unbounded read
            if self.will_close:
                s = self.fp.read()
                s = self.fp.read(self.length)
            self.close()      # we read everything
            return s

        if self.length is not None:
            if amt > self.length:
                # clip the read to the "end of response"
                amt = self.length
            self.length = self.length - amt

        s = self.fp.read(amt)

        # close our "file" if we know we should
        ### I'm not sure about the len(s) < amt part; we should be
        ### safe because we shouldn't be using non-blocking sockets
        if self.length == 0 or len(s) < amt:

        return s

class HTTPConnection:

    _http_vsn = 11
    _http_vsn_str = 'HTTP/1.1'

    response_class = HTTPResponse
    default_port = HTTP_PORT

    def __init__(self, host, port=None):
        self.sock = None
        self.response = None
        self._set_hostport(host, port)

    def _set_hostport(self, host, port):
        if port is None:
            i = string.find(host, ':')
            if i >= 0:
                port = int(host[i+1:])
                host = host[:i]
                port = self.default_port
        self.host = host
        self.port = port
        self.addr = host, port

    def connect(self):
        """Connect to the host and port specified in __init__."""
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

    def close(self):
        """Close the connection to the HTTP server."""
        if self.sock:
            self.sock.close() # close it manually... there may be other refs
            self.sock = None
        if self.response:
            self.response = None

    def send(self, str):
        """Send `str' to the server."""
        if self.sock is None:

        # send the data to the server. if we get a broken pipe, then close
        # the socket. we want to reconnect when somebody tries to send again.
        # NOTE: we DO propagate the error, though, because we cannot simply
        #       ignore the error... the caller will know if they can retry.
        except socket.error, v:
            if v[0] == 32:    # Broken pipe

    def putrequest(self, method, url):
        """Send a request to the server.

        `method' specifies an HTTP request method, e.g. 'GET'.
        `url' specifies the object being requested, e.g.
        if self.response is not None:
            if not self.response.isclosed():
                ### implies half-duplex!
                raise error, 'prior response has not been fully handled'
            self.response = None

        if not url:
            url = '/'
        str = '%s %s %s\r\n' % (method, url, self._http_vsn_str)

        except socket.error, v:
            if v[0] != 32:    # Broken pipe
            # try one more time (the socket was closed; this will reopen)

        self.putheader('Host', self.host)

        if self._http_vsn == 11:
            # Issue some standard headers for better HTTP/1.1 compliance

            # note: we are assuming that clients will not attempt to set these
            #     headers since *this* library must deal with the consequences.
            #     this also means that when the supporting libraries are
            #     updated to recognize other forms, then this code should be
            #     changed (removed or updated).

            # we only want a Content-Encoding of "identity" since we don't
            # support encodings such as x-gzip or x-deflate.
            self.putheader('Accept-Encoding', 'identity')

            # we can accept "chunked" Transfer-Encodings, but no others
            # NOTE: no TE header implies *only* "chunked"
            #self.putheader('TE', 'chunked')

            # if TE is supplied in the header, then it must appear in a
            # Connection header.
            #self.putheader('Connection', 'TE')

            # For HTTP/1.0, the server will assume "not chunked"

    def putheader(self, header, value):
        """Send a request header line to the server.

        For example: h.putheader('Accept', 'text/html')
        str = '%s: %s\r\n' % (header, value)

    def endheaders(self):
        """Indicate that the last header line has been sent to the server."""


    def request(self, method, url, body=None, headers={}):
        """Send a complete request to the server."""

            self._send_request(method, url, body, headers)
        except socket.error, v:
            if v[0] != 32:    # Broken pipe
            # try one more time
            self._send_request(method, url, body, headers)

    def _send_request(self, method, url, body, headers):
        self.putrequest(method, url)

        if body:
            self.putheader('Content-Length', str(len(body)))
        for hdr, value in headers.items():
            self.putheader(hdr, value)

        if body:

    def getreply(self):
        """Get a reply from the server.

        Returns a tuple consisting of:
        - server response code (e.g. '200' if all goes well)
        - server response string corresponding to response code
        - any RFC822 headers in the response from the server

        file = self.sock.makefile('rb')
        line = file.readline()
            [ver, code, msg] = string.split(line, None, 2)
        except ValueError:
                [ver, code] = string.split(line, None, 1)
                msg = ""
            except ValueError:
                return -1, line, file
        if ver[:5] != 'HTTP/':
            return -1, line, file
        errcode = int(code)
        errmsg = string.strip(msg)
        response = self.response_class(file, ver, errcode)
        if response.will_close:
            # this effectively passes the connection to the response
            # remember this, so we can tell when it is complete
            self.response = response
        return errcode, errmsg, response

class FakeSocket:
    def __init__(self, sock, ssl):
        self.__sock = sock
        self.__ssl = ssl

    def makefile(self, mode):           # hopefully, never have to write
        # XXX add assert about mode != w???
        msgbuf = ""
        while 1:
                msgbuf = msgbuf + self.__ssl.read()
            except socket.sslerror, msg:
        return StringIO(msgbuf)

    def send(self, stuff, flags = 0):
        return self.__ssl.write(stuff)

    def recv(self, len = 1024, flags = 0):
        return self.__ssl.read(len)

    def __getattr__(self, attr):
        return getattr(self.__sock, attr)

class HTTPSConnection(HTTPConnection):
    """This class allows communication via SSL."""
    __super_init = HTTPConnection.__init__

    default_port = HTTPS_PORT

    def __init__(self, host, port=None, **x509):
        self.__super_init(host, port)
        self.key_file = x509.get('key_file')
        self.cert_file = x509.get('cert_file')

    def connect(self):
        """Connect to a host onf a given port
        Note: This method is automatically invoked by __init__, if a host
        is specified during instantiation.
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        ssl = socket.ssl(sock, self.key_file, self.cert_file)
        self.sock = FakeSocket(sock, ssl)

class HTTPMixin:
    """Mixin for compatibility with httplib.py from 1.5.

    requires that class that inherits defines the following attributes:

    _http_vsn = 10
    _http_vsn_str = 'HTTP/1.0'

    def connect(self, host=None, port=None):
        "Accept arguments to set the host/port, since the superclass doesn't."
        if host is not None:
            self._set_hostport(host, port)

    def set_debuglevel(self, debuglevel):
        "The class no longer supports the debuglevel."

    def getfile(self):
        "Provide a getfile, since the superclass' use of HTTP/1.1 prevents it."
        return self.file

    def putheader(self, header, *values):
        "The superclass allows only one value argument."
        self.super_putheader(header, string.joinfields(values,'\r\n\t'))

    def getreply(self):
        "Compensate for an instance attribute shuffling."
        errcode, errmsg, response = self.super_getreply()
        if errcode == -1:
            self.file = response  # response is the "file" when errcode==-1
            self.headers = None
            return -1, errmsg, None

        self.headers = response
        self.file = response.fp
        return errcode, errmsg, response

class HTTP(HTTPMixin, HTTPConnection):
    super_init = HTTPConnection.__init__
    super_connect = HTTPConnection.connect
    super_putheader = HTTPConnection.putheader
    super_getreply = HTTPConnection.getreply

    _http_vsn = 10
    _http_vsn_str = 'HTTP/1.0'

    def __init__(self, host='', port=None):
        "Provide a default host, since the superclass requires one."
        # Note that we may pass an empty string as the host; this will throw
        # an error when we attempt to connect. Presumably, the client code
        # will call connect before then, with a proper host.
        self.super_init(host, port)

class HTTPS(HTTPMixin, HTTPSConnection):
    super_init = HTTPSConnection.__init__
    super_connect = HTTPSConnection.connect
    super_putheader = HTTPSConnection.putheader
    super_getreply = HTTPSConnection.getreply

    _http_vsn = 10
    _http_vsn_str = 'HTTP/1.0'

    def __init__(self, host='', port=None, **x509):
        "Provide a default host, since the superclass requires one."
        # Note that we may pass an empty string as the host; this will throw
        # an error when we attempt to connect. Presumably, the client code
        # will call connect before then, with a proper host.
        self.super_init(host, port, **x509)

def test():
    """Test this module.

    The test consists of retrieving and displaying the Python
    home page, along with the error code and error string returned
    by the www.python.org server.

    import sys
    import getopt
    opts, args = getopt.getopt(sys.argv[1:], 'd')
    dl = 0
    for o, a in opts:
        if o == '-d': dl = dl + 1
    host = 'www.python.org'
    selector = '/'
    if args[0:]: host = args[0]
    if args[1:]: selector = args[1]
    h = HTTP()
    h.putrequest('GET', selector)
    errcode, errmsg, headers = h.getreply()
    print 'errcode =', errcode
    print 'errmsg  =', errmsg
    if headers:
        for header in headers.headers: print string.strip(header)
    print h.getfile().read()

    if hasattr(socket, 'ssl'):
        host = 'www.c2.net'
        hs = HTTPS()
        hs.putrequest('GET', selector)
        errcode, errmsg, headers = hs.getreply()
        print 'errcode =', errcode
        print 'errmsg  =', errmsg
        if headers:
            for header in headers.headers: print string.strip(header)
        print hs.getfile().read()

if __name__ == '__main__':