From 55f9fc1295ab548745247d1a6a7b391d5a94877f Mon Sep 17 00:00:00 2001 From: Hiran Date: Thu, 7 Apr 2022 15:00:46 -0400 Subject: [PATCH 1/3] content length mismatch fix --- starlette/middleware/gzip.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/starlette/middleware/gzip.py b/starlette/middleware/gzip.py index 9d69ee7ca..46c5c7ed1 100644 --- a/starlette/middleware/gzip.py +++ b/starlette/middleware/gzip.py @@ -77,8 +77,9 @@ async def send_with_gzip(self, message: Message) -> None: headers.add_vary_header("Accept-Encoding") del headers["Content-Length"] - self.gzip_file.write(body) - message["body"] = self.gzip_buffer.getvalue() + if body and body != b"": + self.gzip_file.write(body) + message["body"] = self._set_response_body(body, more_body) self.gzip_buffer.seek(0) self.gzip_buffer.truncate() @@ -90,16 +91,33 @@ async def send_with_gzip(self, message: Message) -> None: body = message.get("body", b"") more_body = message.get("more_body", False) - self.gzip_file.write(body) + if body and body != b"": + self.gzip_file.write(body) if not more_body: self.gzip_file.close() - message["body"] = self.gzip_buffer.getvalue() + message["body"] = self._set_response_body(body, more_body) self.gzip_buffer.seek(0) self.gzip_buffer.truncate() await self.send(message) + def _set_response_body(self, body, more_body: bool = True) -> bytes: + """Null byte fix + + If the response body is null and we try to get the buffervalue, a null byte is produced. + This causes a content length mismatch. + + This issues exists in the upstream Gzip middleware in starlette. + - https://github.com/tiangolo/fastapi/issues/4050 + - https://github.com/tiangolo/fastapi/issues/2818 + + """ + # Check # [b"", 0x1f, 0x8b, 0xff] is the header for a gzip file. + if body == b"" and more_body: + return body + return self.gzip_buffer.getvalue() + async def unattached_send(message: Message) -> typing.NoReturn: raise RuntimeError("send awaitable not set") # pragma: no cover From 082d3a7fc4aa58b2c74e2c038f853897b845f5bf Mon Sep 17 00:00:00 2001 From: Hiran Date: Fri, 8 Apr 2022 20:50:18 -0400 Subject: [PATCH 2/3] fix: better handling of null body --- starlette/middleware/gzip.py | 37 +++++++++++++----------------------- 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/starlette/middleware/gzip.py b/starlette/middleware/gzip.py index 46c5c7ed1..44cf0bf94 100644 --- a/starlette/middleware/gzip.py +++ b/starlette/middleware/gzip.py @@ -26,7 +26,7 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: await self.app(scope, receive, send) -class GZipResponder: +class GZipResponder: # noqa: WPS230 def __init__(self, app: ASGIApp, minimum_size: int, compresslevel: int = 9) -> None: self.app = app self.minimum_size = minimum_size @@ -35,8 +35,7 @@ def __init__(self, app: ASGIApp, minimum_size: int, compresslevel: int = 9) -> N self.started = False self.gzip_buffer = io.BytesIO() self.gzip_file = gzip.GzipFile( - mode="wb", fileobj=self.gzip_buffer, compresslevel=compresslevel - ) + mode="wb", fileobj=self.gzip_buffer, compresslevel=compresslevel) async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: self.send = send @@ -56,11 +55,9 @@ async def send_with_gzip(self, message: Message) -> None: # Don't apply GZip to small outgoing responses. await self.send(self.initial_message) await self.send(message) - elif not more_body: + elif not more_body: # noqa: WPS504 # Standard GZip response. - self.gzip_file.write(body) - self.gzip_file.close() - body = self.gzip_buffer.getvalue() + body = self._set_response_body(body, more_body) headers = MutableHeaders(raw=self.initial_message["headers"]) headers["Content-Encoding"] = "gzip" @@ -75,14 +72,9 @@ async def send_with_gzip(self, message: Message) -> None: headers = MutableHeaders(raw=self.initial_message["headers"]) headers["Content-Encoding"] = "gzip" headers.add_vary_header("Accept-Encoding") - del headers["Content-Length"] + del headers["Content-Length"] # noqa: WPS420 - if body and body != b"": - self.gzip_file.write(body) message["body"] = self._set_response_body(body, more_body) - self.gzip_buffer.seek(0) - self.gzip_buffer.truncate() - await self.send(self.initial_message) await self.send(message) @@ -91,14 +83,7 @@ async def send_with_gzip(self, message: Message) -> None: body = message.get("body", b"") more_body = message.get("more_body", False) - if body and body != b"": - self.gzip_file.write(body) - if not more_body: - self.gzip_file.close() - message["body"] = self._set_response_body(body, more_body) - self.gzip_buffer.seek(0) - self.gzip_buffer.truncate() await self.send(message) @@ -113,10 +98,14 @@ def _set_response_body(self, body, more_body: bool = True) -> bytes: - https://github.com/tiangolo/fastapi/issues/2818 """ - # Check # [b"", 0x1f, 0x8b, 0xff] is the header for a gzip file. - if body == b"" and more_body: - return body - return self.gzip_buffer.getvalue() + if body and body not in {b"", b"null"}: + self.gzip_file.write(body) + if not more_body: + self.gzip_file.close() + value = self.gzip_buffer.getvalue() + self.gzip_buffer.seek(0) + self.gzip_buffer.truncate() + return value async def unattached_send(message: Message) -> typing.NoReturn: From 3019f31f870481da0681443c451d65dd3e2d9789 Mon Sep 17 00:00:00 2001 From: Marcelo Trylesinski Date: Mon, 25 Apr 2022 14:36:35 +0200 Subject: [PATCH 3/3] Apply suggestions from code review --- starlette/middleware/gzip.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/starlette/middleware/gzip.py b/starlette/middleware/gzip.py index 44cf0bf94..5949cfc93 100644 --- a/starlette/middleware/gzip.py +++ b/starlette/middleware/gzip.py @@ -26,7 +26,7 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: await self.app(scope, receive, send) -class GZipResponder: # noqa: WPS230 +class GZipResponder: def __init__(self, app: ASGIApp, minimum_size: int, compresslevel: int = 9) -> None: self.app = app self.minimum_size = minimum_size @@ -35,7 +35,8 @@ def __init__(self, app: ASGIApp, minimum_size: int, compresslevel: int = 9) -> N self.started = False self.gzip_buffer = io.BytesIO() self.gzip_file = gzip.GzipFile( - mode="wb", fileobj=self.gzip_buffer, compresslevel=compresslevel) + mode="wb", fileobj=self.gzip_buffer, compresslevel=compresslevel + ) async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: self.send = send @@ -55,7 +56,7 @@ async def send_with_gzip(self, message: Message) -> None: # Don't apply GZip to small outgoing responses. await self.send(self.initial_message) await self.send(message) - elif not more_body: # noqa: WPS504 + elif not more_body: # Standard GZip response. body = self._set_response_body(body, more_body) @@ -72,7 +73,7 @@ async def send_with_gzip(self, message: Message) -> None: headers = MutableHeaders(raw=self.initial_message["headers"]) headers["Content-Encoding"] = "gzip" headers.add_vary_header("Accept-Encoding") - del headers["Content-Length"] # noqa: WPS420 + del headers["Content-Length"] message["body"] = self._set_response_body(body, more_body) await self.send(self.initial_message)