remove whitespace in transfer-encoding

gunicorn/http/message.py

@@ -135,6 +135,9 @@ class PPProtocol(IntEnum):
 # control range (0x00-0x1F except HTAB, plus DEL 0x7F) must be rejected.
 RFC9110_5_5_INVALID_AND_DANGEROUS = re.compile(r"[\x00-\x08\x0a-\x1f\x7f]")
 
+# OWS = *( SP / HTAB )
+RFC9110_5_6_3_WHITESPACE = " \t"
+
 # RFC 9110 section 6.5.1: fields forbidden in trailers because they alter
 # routing, framing, or authentication. Using the uppercased names stored
 # by parse_headers.
@@ -308,6 +311,7 @@ def parse_headers(self, data, from_trailer=False):
                 raise InvalidHeader(curr)
             name, value = curr.split(":", 1)
             if self.cfg.strip_header_spaces:
+                # non-standard and dangerous
                 name = name.rstrip(" \t")
             if not TOKEN_RE.fullmatch(name):
                 raise InvalidHeaderName(name)
@@ -322,10 +326,12 @@ def parse_headers(self, data, from_trailer=False):
             if from_trailer and name in RFC9110_6_5_1_FORBIDDEN_TRAILER:
                 raise InvalidHeaderName(name)
 
-            value = [value.strip(" \t")]
+            # https://datatracker.ietf.org/doc/html/rfc9112#name-field-syntax
+            # optional whitespace before and after field-value
+            value = [value.strip(RFC9110_5_6_3_WHITESPACE)]
 
             # Consume value continuation lines..
-            while lines and lines[0].startswith((" ", "\t")):
+            while lines and lines[0].startswith(tuple(RFC9110_5_6_3_WHITESPACE)):
                 # .. which is obsolete here, and no longer done by default
                 if not self.cfg.permit_obsolete_folding:
                     raise ObsoleteFolding(name)
@@ -334,7 +340,7 @@ def parse_headers(self, data, from_trailer=False):
                 if header_length > self.limit_request_field_size > 0:
                     raise LimitRequestHeaders("limit request headers "
                                               "fields size")
-                value.append(curr.strip("\t "))
+                value.append(curr.strip(RFC9110_5_6_3_WHITESPACE))
             value = " ".join(value)
 
             if RFC9110_5_5_INVALID_AND_DANGEROUS.search(value):
@@ -366,7 +372,7 @@ def set_body_reader(self):
             elif name == "TRANSFER-ENCODING":
                 # T-E can be a list
                 # https://datatracker.ietf.org/doc/html/rfc9112#name-transfer-encoding
-                vals = [v.strip() for v in value.split(',')]
+                vals = [v.strip(RFC9110_5_6_3_WHITESPACE) for v in value.split(',')]
                 for val in vals:
                     if val.lower() == "chunked":
                         # DANGER: transfer codings stack, and stacked chunking is never intended
@@ -419,7 +425,9 @@ def should_close(self):
             return True
         for (h, v) in self.headers:
             if h == "CONNECTION":
-                v = v.lower().strip(" \t")
+                # https://datatracker.ietf.org/doc/html/rfc9110#section-7.6.1
+                # Connection options are case-insensitive.
+                v = v.lower().strip(RFC9110_5_6_3_WHITESPACE)
                 if v == "close":
                     return True
                 elif v == "keep-alive":

gunicorn/util.py

@@ -32,6 +32,7 @@
 
 REDIRECT_TO = getattr(os, 'devnull', '/dev/null')
 
+# https://datatracker.ietf.org/doc/html/rfc9110#section-7.6.1
 # Server and Date aren't technically hop-by-hop
 # headers, but they are in the purview of the
 # origin server which the WSGI spec says we should
@@ -41,6 +42,7 @@
 # might be better, but nothing else does it and
 # dropping them is easier.
 hop_headers = set("""
+    proxy-connection
     connection keep-alive proxy-authenticate proxy-authorization
     te trailers transfer-encoding upgrade
     server date

tests/requests/invalid/chunked_whitespace.http

@@ -0,0 +1,8 @@
+POST /upload HTTP/1.1\r\n
+Host: example.com\r\n
+Transfer-Encoding: identity, \x0bchunked\r\n
+\r\n
+5\r\n
+hello\r\n
+0\r\n
+\r\n

tests/requests/invalid/chunked_whitespace.py

@@ -0,0 +1,6 @@
+#
+# This file is part of gunicorn released under the MIT license.
+# See the NOTICE for more information.
+
+from gunicorn.http.errors import InvalidHeader
+request = InvalidHeader

tests/requests/valid/obs_fold_whitespace.http

@@ -0,0 +1,18 @@
+POST /1 HTTP/1.1\r\n
+Host: example.com\r\n
+Connection: keep-alive\r\n
+ \r\n
+Content-Length: 3\r\n
+\r\n
+123
+POST /2 HTTP/1.1\r\n
+Host: example.com\r\n
+Connection: close\r\n
+ \r\n
+Transfer-Encoding: chunked\r\n
+\r\n
+3\r\n
+123\r\n
+0\r\n
+Trailer-Checksum: 0\r\n
+\r\n

tests/requests/valid/obs_fold_whitespace.py

@@ -0,0 +1,38 @@
+#
+# This file is part of gunicorn released under the MIT license.
+# See the NOTICE for more information.
+
+from gunicorn.http.errors import ObsoleteFolding
+from gunicorn.config import Config
+
+cfg = Config()
+cfg.set('permit_obsolete_folding', True)
+
+req1 = {
+    "method": "POST",
+    "uri": uri("/1"),
+    "version": (1, 1),
+    "headers": [
+        ("HOST", "example.com"),
+        ("CONNECTION", "keep-alive "),
+        ("CONTENT-LENGTH", "3"),
+    ],
+    "body": b"123",
+}
+
+req2 = {
+    "method": "POST",
+    "uri": uri("/2"),
+    "version": (1, 1),
+    "headers": [
+        ("HOST", "example.com"),
+        ("CONNECTION", "close "),
+        ("TRANSFER-ENCODING", "chunked"),
+    ],
+    "trailers": [
+        ("TRAILER-CHECKSUM", "0"),
+    ],
+    "body": b"123",
+}
+
+request = [req1, req2]