http.client源码刨析

# HTTPMessage, parse_headers(), and the HTTP status code constants are# intentionally omitted for simplicity__all__ = ["HTTPResponse", "HTTPConnection",           "HTTPException", "NotConnected", "UnknownProtocol",           "UnknownTransferEncoding", "UnimplementedFileMode",           "IncompleteRead", "InvalidURL", "ImproperConnectionState",           "CannotSendRequest", "CannotSendHeader", "ResponseNotReady",           "BadStatusLine", "LineTooLong", "RemoteDisconnected", "error",           "responses"]

HTTP_PORT = 80HTTPS_PORT = 443

_UNKNOWN = 'UNKNOWN'
# connection states_CS_IDLE = 'Idle'_CS_REQ_STARTED = 'Request-started'_CS_REQ_SENT = 'Request-sent'

class HTTPResponse(io.BufferedIOBase):    def __init__(self, sock, debuglevel=0, method=None, url=None):        self.fp = sock.makefile("rb")        self.debuglevel = debuglevel        self._method = method
        self.headers = self.msg = None
        # from the Status-Line of the response        self.version = _UNKNOWN # HTTP-Version        self.status = _UNKNOWN  # Status-Code        self.reason = _UNKNOWN  # Reason-Phrase
        self.chunked = _UNKNOWN         # is "chunked" being used?        self.chunk_left = _UNKNOWN      # bytes left to read in current chunk        self.length = _UNKNOWN          # number of bytes left in response        self.will_close = _UNKNOWN      # conn will close at end of response

class HTTPResponse(io.BufferedIOBase):    def read(self, amt=None):        if self.fp is None:            return b""
        if self._method == "HEAD":            self._close_conn()            return b""
        if amt is not None:            # Amount is given, implement using readinto            b = bytearray(amt)            n = self.readinto(b)            return memoryview(b)[:n].tobytes()        else:            # Amount is not given (unbounded read) so we must check self.length            # and self.chunked
            if self.chunked:                return self._readall_chunked()
            if self.length is None:                s = self.fp.read()            else:                try:                    s = self._safe_read(self.length)                except IncompleteRead:                    self._close_conn()                    raise                self.length = 0            self._close_conn()        # we read everything            return s

class HTTPResponse(io.BufferedIOBase):    def begin(self):        if self.headers is not None:            # we've already started reading the response            return
        # read until we get a non-100 response        while True:            version, status, reason = self._read_status()            if status != CONTINUE:                break            # skip the header from the 100 response            while True:                skip = self.fp.readline(_MAXLINE + 1)                if len(skip) > _MAXLINE:                    raise LineTooLong("header line")                skip = skip.strip()                if not skip:                    break                if self.debuglevel > 0:                    print("header:", skip)
        self.code = self.status = status        self.reason = reason.strip()        if version in ("HTTP/1.0", "HTTP/0.9"):            # Some servers might still return "0.9", treat it as 1.0 anyway            self.version = 10        elif version.startswith("HTTP/1."):            self.version = 11   # use HTTP/1.1 code for HTTP/1.x where x>=1        else:            raise UnknownProtocol(version)
        self.headers = self.msg = parse_headers(self.fp)
        if self.debuglevel > 0:            for hdr, val in self.headers.items():                print("header:", hdr + ":", val)
        # are we using the chunked-style of transfer encoding?        tr_enc = self.headers.get("transfer-encoding")        if tr_enc and tr_enc.lower() == "chunked":            self.chunked = True            self.chunk_left = None        else:            self.chunked = False
        # will the connection close at the end of the response?        self.will_close = self._check_close()
        # do we have a Content-Length?        # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"        self.length = None        length = self.headers.get("content-length")
         # are we using the chunked-style of transfer encoding?        tr_enc = self.headers.get("transfer-encoding")        if length and not self.chunked:            try:                self.length = int(length)            except ValueError:                self.length = None            else:                if self.length < 0:  # ignore nonsensical negative lengths                    self.length = None        else:            self.length = None
        # does the body have a fixed length? (of zero)        if (status == NO_CONTENT or status == NOT_MODIFIED or            100 <= status < 200 or      # 1xx codes            self._method == "HEAD"):            self.length = 0
        # if the connection remains open, and we aren't using chunked, and        # a content-length was not provided, then assume that the connection        # WILL close.        if (not self.will_close and            not self.chunked and            self.length is None):            self.will_close = True

class HTTPConnection:    _http_vsn = 11    _http_vsn_str = 'HTTP/1.1'
    response_class = HTTPResponse    default_port = 80    auto_open = 1    debuglevel = 0

class HTTPConnection:    def __init__(self, host, port=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,                 source_address=None, blocksize=8192)        ...

class HTTPConnection:    def putrequest(self, method, url, skip_host=False,                   skip_accept_encoding=False):       """Send a request to the server"""
        # if a prior response has been completed, then forget about it.        if self.__response and self.__response.isclosed():            self.__response = None

        # in certain cases, we cannot issue another request on this connection.        # this occurs when:        #   1) we are in the process of sending a request.   (_CS_REQ_STARTED)        #   2) a response to a previous request has signalled that it is going        #      to close the connection upon completion.        #   3) the headers for the previous response have not been read, thus        #      we cannot determine whether point (2) is true.   (_CS_REQ_SENT)        #        # if there is no prior response, then we can request at will.        #        # if point (2) is true, then we will have passed the socket to the        # response (effectively meaning, "there is no prior response"), and        # will open a new one when a new request is made.        #        # Note: if a prior response exists, then we *can* start a new request.        #       We are not allowed to begin fetching the response to this new        #       request, however, until that prior response is complete.        #        if self.__state == _CS_IDLE:            self.__state = _CS_REQ_STARTED        else:            raise CannotSendRequest(self.__state)
        # Save the method for use later in the response phase        self._method = method
        url = url or '/'        self._validate_path(url)
        request = '%s %s %s' % (method, url, self._http_vsn_str)
        self._output(self._encode_request(request))
        if self._http_vsn == 11:            # Issue some standard headers for better HTTP/1.1 compliance
            if not skip_host:                # this header is issued *only* for HTTP/1.1                # connections. more specifically, this means it is                # only issued when the client uses the new                # HTTPConnection() class. backwards-compat clients                # will be using HTTP/1.0 and those clients may be                # issuing this header themselves. we should NOT issue                # it twice; some web servers (such as Apache) barf                # when they see two Host: headers
                # If we need a non-standard port,include it in the                # header.  If the request is going through a proxy,                # but the host of the actual URL, not the host of the                # proxy.
                netloc = ''                if url.startswith('http'):                    nil, netloc, nil, nil, nil = urlsplit(url)
                if netloc:                    try:                        netloc_enc = netloc.encode("ascii")                    except UnicodeEncodeError:                        netloc_enc = netloc.encode("idna")                    self.putheader('Host', netloc_enc)                else:                    if self._tunnel_host:                        host = self._tunnel_host                        port = self._tunnel_port                    else:                        host = self.host                        port = self.port
                    try:                        host_enc = host.encode("ascii")                    except UnicodeEncodeError:                        host_enc = host.encode("idna")
                    # As per RFC 273, IPv6 address should be wrapped with []                    # when used as Host header
                    if host.find(':') >= 0:                        host_enc = b'[' + host_enc + b']'
                    if port == self.default_port:                        self.putheader('Host', host_enc)                    else:                        host_enc = host_enc.decode("ascii")                        self.putheader('Host', "%s:%s" % (host_enc, port))
            # note: we are assuming that clients will not attempt to set these            #       headers since *this* library must deal with the            #       consequences. this also means that when the supporting            #       libraries are updated to recognize other forms, then this            #       code should be changed (removed or updated).
            # we only want a Content-Encoding of "identity" since we don't            # support encodings such as x-gzip or x-deflate.            if not skip_accept_encoding:                self.putheader('Accept-Encoding', 'identity')
            # we can accept "chunked" Transfer-Encodings, but no others            # NOTE: no TE header implies *only* "chunked"            #self.putheader('TE', 'chunked')
            # if TE is supplied in the header, then it must appear in a            # Connection header.            #self.putheader('Connection', 'TE')
        else:            # For HTTP/1.0, the server will assume "not chunked"            pass

class HTTPConnection:    def putheader(self, header, *values):        """Send a request header line to the server.
        For example: h.putheader('Accept', 'text/html')        """        if self.__state != _CS_REQ_STARTED:            raise CannotSendHeader()
        if hasattr(header, 'encode'):            header = header.encode('ascii')
        if not _is_legal_header_name(header):            raise ValueError('Invalid header name %r' % (header,))
        values = list(values)        for i, one_value in enumerate(values):            if hasattr(one_value, 'encode'):                values[i] = one_value.encode('latin-1')            elif isinstance(one_value, int):                values[i] = str(one_value).encode('ascii')
            if _is_illegal_header_value(values[i]):                raise ValueError('Invalid header value %r' % (values[i],))
        value = b'\r\n\t'.join(values)        header = header + b': ' + value        self._output(header)

class HTTPConnection:    def endheaders(self, message_body=None, *, encode_chunked=False):        """Indicate that the last header line has been sent to the server.
        This method sends the request to the server.  The optional message_body        argument can be used to pass a message body associated with the        request.        """        if self.__state == _CS_REQ_STARTED:            self.__state = _CS_REQ_SENT        else:            raise CannotSendHeader()        self._send_output(message_body, encode_chunked=encode_chunked)

class HTTPConnection:    def getresponse(self):        """Get the response from the server.
        If the HTTPConnection is in the correct state, returns an        instance of HTTPResponse or of whatever object is returned by        the response_class variable.
        If a request has not been sent or if a previous response has        not be handled, ResponseNotReady is raised.  If the HTTP        response indicates that the connection should be closed, then        it will be closed before the response is returned.  When the        connection is closed, the underlying socket is closed.        """
        # if a prior response has been completed, then forget about it.        if self.__response and self.__response.isclosed():            self.__response = None
        # if a prior response exists, then it must be completed (otherwise, we        # cannot read this response's header to determine the connection-close        # behavior)        #        # note: if a prior response existed, but was connection-close, then the        # socket and response were made independent of this HTTPConnection        # object since a new request requires that we open a whole new        # connection        #        # this means the prior response had one of two states:        #   1) will_close: this connection was reset and the prior socket and        #                  response operate independently        #   2) persistent: the response was retained and we await its        #                  isclosed() status to become true.        #        if self.__state != _CS_REQ_SENT or self.__response:            raise ResponseNotReady(self.__state)
        if self.debuglevel > 0:            response = self.response_class(self.sock, self.debuglevel,                                           method=self._method)        else:            response = self.response_class(self.sock, method=self._method)
        try:            try:                response.begin()            except ConnectionError:                self.close()                raise            assert response.will_close != _UNKNOWN            self.__state = _CS_IDLE
            if response.will_close:                # this effectively passes the connection to the response                self.close()            else:                # remember this, so we can tell when it is complete                self.__response = response
            return response        except:            response.close()            raise

class HTTPConnection:    def request(self, method, url, body=None, headers={}, *,                encode_chunked=False):        """Send a complete request to the server."""        self._send_request(method, url, body, headers, encode_chunked)