Raw input headers

aiohttp/client_reqrep.py

@@ -514,6 +514,7 @@ class ClientResponse:
     cookies = None  # Response cookies (Set-Cookie)
     content = None  # Payload stream
     headers = None  # Response headers, CIMultiDictProxy
+    raw_headers = None  # Response raw headers, a sequence of pairs
 
     _connection = None  # current connection
     flow_control_class = FlowControlStreamReader  # reader flow control
@@ -610,6 +611,7 @@ def start(self, connection, read_until_eof=False):
 
         # headers
         self.headers = CIMultiDictProxy(message.headers)
+        self.raw_headers = tuple(message.raw_headers)
 
         # payload
         response_with_body = self._need_parse_response_body()

aiohttp/errors.py

@@ -145,6 +145,8 @@ def __init__(self, line, limit='Unknown'):
 class InvalidHeader(BadHttpMessage):
 
     def __init__(self, hdr):
+        if isinstance(hdr, bytes):
+            hdr = hdr.decode('utf-8', 'surrogateescape')
         super().__init__('Invalid HTTP Header: {}'.format(hdr))
         self.hdr = hdr
 

aiohttp/multipart.py

@@ -558,10 +558,10 @@ def _read_boundary(self):
 
     @asyncio.coroutine
     def _read_headers(self):
-        lines = ['']
+        lines = [b'']
         while True:
             chunk = yield from self._content.readline()
-            chunk = chunk.decode().strip()
+            chunk = chunk.strip()
             lines.append(chunk)
             if not chunk:
                 break

aiohttp/protocol.py

@@ -25,8 +25,8 @@
 ASCIISET = set(string.printable)
 METHRE = re.compile('[A-Z0-9$-_.]+')
 VERSRE = re.compile('HTTP/(\d+).(\d+)')
-HDRRE = re.compile('[\x00-\x1F\x7F()<>@,;:\[\]={} \t\\\\\"]')
-CONTINUATION = (' ', '\t')
+HDRRE = re.compile(b'[\x00-\x1F\x7F()<>@,;:\[\]={} \t\\\\\"]')
+CONTINUATION = (32, 9)  # (' ', '\t')
 EOF_MARKER = object()
 EOL_MARKER = object()
 STATUS_LINE_READY = object()
@@ -43,12 +43,14 @@
 
 RawRequestMessage = collections.namedtuple(
     'RawRequestMessage',
-    ['method', 'path', 'version', 'headers', 'should_close', 'compression'])
+    ['method', 'path', 'version', 'headers', 'raw_headers',
+     'should_close', 'compression'])
 
 
 RawResponseMessage = collections.namedtuple(
     'RawResponseMessage',
-    ['version', 'code', 'reason', 'headers', 'should_close', 'compression'])
+    ['version', 'code', 'reason', 'headers', 'raw_headers',
+     'should_close', 'compression'])
 
 
 class HttpParser:
@@ -60,14 +62,15 @@ def __init__(self, max_line_size=8190, max_headers=32768,
         self.max_field_size = max_field_size
 
     def parse_headers(self, lines):
-        """Parses RFC2822 headers from a stream.
+        """Parses RFC 5322 headers from a stream.
 
         Line continuations are supported. Returns list of header name
         and value pairs. Header name is in upper case.
         """
         close_conn = None
         encoding = None
         headers = CIMultiDict()
+        raw_headers = []
 
         lines_idx = 1
         line = lines[1]
@@ -77,13 +80,13 @@ def parse_headers(self, lines):
 
             # Parse initial header name : value pair.
             try:
-                name, value = line.split(':', 1)
+                bname, bvalue = line.split(b':', 1)
             except ValueError:
                 raise errors.InvalidHeader(line) from None
 
-            name = name.strip(' \t').upper()
-            if HDRRE.search(name):
-                raise errors.InvalidHeader(name)
+            bname = bname.strip(b' \t').upper()
+            if HDRRE.search(bname):
+                raise errors.InvalidHeader(bname)
 
             # next line
             lines_idx += 1
@@ -93,25 +96,28 @@ def parse_headers(self, lines):
             continuation = line and line[0] in CONTINUATION
 
             if continuation:
-                value = [value]
+                bvalue = [bvalue]
                 while continuation:
                     header_length += len(line)
                     if header_length > self.max_field_size:
                         raise errors.LineTooLong(
                             'limit request headers fields size')
-                    value.append(line)
+                    bvalue.append(line)
 
                     # next line
                     lines_idx += 1
                     line = lines[lines_idx]
                     continuation = line[0] in CONTINUATION
-                value = '\r\n'.join(value)
+                bvalue = b'\r\n'.join(bvalue)
             else:
                 if header_length > self.max_field_size:
                     raise errors.LineTooLong(
                         'limit request headers fields size')
 
-            value = value.strip()
+            bvalue = bvalue.strip()
+
+            name = bname.decode('utf-8', 'surrogateescape')
+            value = bvalue.decode('utf-8', 'surrogateescape')
 
             # keep-alive and encoding
             if name == hdrs.CONNECTION:
@@ -126,8 +132,9 @@ def parse_headers(self, lines):
                     encoding = enc
 
             headers.add(name, value)
+            raw_headers.append((bname, bvalue))
 
-        return headers, close_conn, encoding
+        return headers, raw_headers, close_conn, encoding
 
 
 class HttpPrefixParser:
@@ -167,11 +174,10 @@ def __call__(self, out, buf):
         except errors.LineLimitExceededParserError as exc:
             raise errors.LineTooLong(exc.limit) from None
 
-        lines = raw_data.decode(
-            'utf-8', 'surrogateescape').split('\r\n')
+        lines = raw_data.split(b'\r\n')
 
         # request line
-        line = lines[0]
+        line = lines[0].decode('utf-8', 'surrogateescape')
         try:
             method, path, version = line.split(None, 2)
         except ValueError:
@@ -193,7 +199,7 @@ def __call__(self, out, buf):
             raise errors.BadStatusLine(version)
 
         # read headers
-        headers, close, compression = self.parse_headers(lines)
+        headers, raw_headers, close, compression = self.parse_headers(lines)
         if close is None:  # then the headers weren't set in the request
             if version <= HttpVersion10:  # HTTP 1.0 must asks to not close
                 close = True
@@ -202,7 +208,8 @@ def __call__(self, out, buf):
 
         out.feed_data(
             RawRequestMessage(
-                method, path, version, headers, close, compression),
+                method, path, version, headers, raw_headers,
+                close, compression),
             len(raw_data))
         out.feed_eof()
 
@@ -221,10 +228,9 @@ def __call__(self, out, buf):
         except errors.LineLimitExceededParserError as exc:
             raise errors.LineTooLong(exc.limit) from None
 
-        lines = raw_data.decode(
-            'utf-8', 'surrogateescape').split('\r\n')
+        lines = raw_data.split(b'\r\n')
 
-        line = lines[0]
+        line = lines[0].decode('utf-8', 'surrogateescape')
         try:
             version, status = line.split(None, 1)
         except ValueError:
@@ -251,15 +257,15 @@ def __call__(self, out, buf):
             raise errors.BadStatusLine(line)
 
         # read headers
-        headers, close, compression = self.parse_headers(lines)
+        headers, raw_headers, close, compression = self.parse_headers(lines)
 
         if close is None:
             close = version <= HttpVersion10
 
         out.feed_data(
             RawResponseMessage(
                 version, status, reason.strip(),
-                headers, close, compression),
+                headers, raw_headers, close, compression),
             len(raw_data))
         out.feed_eof()
 

aiohttp/web_reqrep.py

@@ -99,29 +99,20 @@ class Request(dict, HeadersMixin):
                     hdrs.METH_TRACE, hdrs.METH_DELETE}
 
     def __init__(self, app, message, payload, transport, reader, writer, *,
-                 _HOST=hdrs.HOST, secure_proxy_ssl_header=None):
+                 secure_proxy_ssl_header=None):
         self._app = app
-        self._version = message.version
+        self._message = message
         self._transport = transport
         self._reader = reader
         self._writer = writer
-        self._method = message.method
-        self._host = message.headers.get(_HOST)
-        self._path_qs = message.path
         self._post = None
         self._post_files_cache = None
-        self._headers = CIMultiDictProxy(message.headers)
-        if self._version < HttpVersion10:
-            self._keep_alive = False
-        else:
-            self._keep_alive = not message.should_close
 
         # matchdict, route_name, handler
         # or information about traversal lookup
         self._match_info = None  # initialized after route resolving
 
         self._payload = payload
-        self._cookies = None
 
         self._read_bytes = None
         self._has_body = not payload.at_eof()
@@ -139,48 +130,48 @@ def scheme(self):
         secure_proxy_ssl_header = self._secure_proxy_ssl_header
         if secure_proxy_ssl_header is not None:
             header, value = secure_proxy_ssl_header
-            if self._headers.get(header) == value:
+            if self.headers.get(header) == value:
                 return 'https'
         return 'http'
 
-    @property
+    @reify
     def method(self):
         """Read only property for getting HTTP method.
 
         The value is upper-cased str like 'GET', 'POST', 'PUT' etc.
         """
-        return self._method
+        return self._message.method
 
-    @property
+    @reify
     def version(self):
         """Read only property for getting HTTP version of request.
 
         Returns aiohttp.protocol.HttpVersion instance.
         """
-        return self._version
+        return self._message.version
 
-    @property
+    @reify
     def host(self):
         """Read only property for getting *HOST* header of request.
 
         Returns str or None if HTTP request has no HOST header.
         """
-        return self._host
+        return self._message.headers.get(hdrs.HOST)
 
-    @property
+    @reify
     def path_qs(self):
         """The URL including PATH_INFO and the query string.
 
         E.g, /app/blog?id=10
         """
-        return self._path_qs
+        return self._message.path
 
     @reify
     def _splitted_path(self):
-        url = '{}://{}{}'.format(self.scheme, self.host, self._path_qs)
+        url = '{}://{}{}'.format(self.scheme, self.host, self.path_qs)
         return urlsplit(url)
 
-    @property
+    @reify
     def raw_path(self):
         """ The URL including raw *PATH INFO* without the host or scheme.
         Warning, the path is unquoted and may contains non valid URL characters
@@ -224,12 +215,17 @@ def POST(self):
             raise RuntimeError("POST is not available before post()")
         return self._post
 
-    @property
+    @reify
     def headers(self):
         """A case-insensitive multidict proxy with all headers."""
-        return self._headers
+        return CIMultiDictProxy(self._message.headers)
 
-    @property
+    @reify
+    def raw_headers(self):
+        """A sequence of pars for all headers."""
+        return tuple(self._message.raw_headers)
+
+    @reify
     def if_modified_since(self, _IF_MODIFIED_SINCE=hdrs.IF_MODIFIED_SINCE):
         """The value of If-Modified-Since HTTP header, or None.
 
@@ -243,10 +239,13 @@ def if_modified_since(self, _IF_MODIFIED_SINCE=hdrs.IF_MODIFIED_SINCE):
                                          tzinfo=datetime.timezone.utc)
         return None
 
-    @property
+    @reify
     def keep_alive(self):
         """Is keepalive enabled by client?"""
-        return self._keep_alive
+        if self.version < HttpVersion10:
+            return False
+        else:
+            return not self._message.should_close
 
     @property
     def match_info(self):
@@ -263,18 +262,16 @@ def transport(self):
         """Transport used for request processing."""
         return self._transport
 
-    @property
+    @reify
     def cookies(self):
         """Return request cookies.
 
         A read-only dictionary-like object.
         """
-        if self._cookies is None:
-            raw = self.headers.get(hdrs.COOKIE, '')
-            parsed = http.cookies.SimpleCookie(raw)
-            self._cookies = MappingProxyType(
-                {key: val.value for key, val in parsed.items()})
-        return self._cookies
+        raw = self.headers.get(hdrs.COOKIE, '')
+        parsed = http.cookies.SimpleCookie(raw)
+        return MappingProxyType(
+            {key: val.value for key, val in parsed.items()})
 
     @property
     def payload(self):

docs/client.rst

@@ -501,7 +501,8 @@ We can check the response status code::
 Response Headers
 ----------------
 
-We can view the server's response headers using a multidict::
+We can view the server's response :attr:`ClientResponse.headers` using
+a :class:`CIMultiDictProxy`::
 
     >>> r.headers
     {'ACCESS-CONTROL-ALLOW-ORIGIN': '*',
@@ -525,6 +526,19 @@ So, we can access the headers using any capitalization we want::
     >>> r.headers.get('content-type')
     'application/json'
 
+All headers converted from binary data using UTF-8 with
+``surrogateescape`` option. That works fine on most cases but
+sometimes unconverted data is needed if a server uses nonstandard
+encoding. While these headers are malformed from :rfc:`7230`
+perspective they are may be retrieved by using
+:attr:`ClientResponse.raw_headers` property::
+
+    >>> r.raw_headers
+    ((b'SERVER', b'nginx'),
+     (b'DATE', b'Sat, 09 Jan 2016 20:28:40 GMT'),
+     (b'CONTENT-TYPE', b'text/html; charset=utf-8'),
+     (b'CONTENT-LENGTH', b'12150'),
+     (b'CONNECTION', b'keep-alive'))
 
 Response Cookies
 ----------------