From dbdc4ab0e6c0e9455221d6b76691e88f7d1a4b8f Mon Sep 17 00:00:00 2001 From: Joshix-1 Date: Sun, 19 Feb 2023 21:00:00 +0000 Subject: [PATCH 1/2] add brotli support --- setup.py | 6 ++ tornado/web.py | 221 +++++++++++++++++++++++++++++++++++-------------- 2 files changed, 167 insertions(+), 60 deletions(-) diff --git a/setup.py b/setup.py index 4a67a715b7..9e4aa84953 100644 --- a/setup.py +++ b/setup.py @@ -119,5 +119,11 @@ def get_tag(self): "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", ], + extras_require={ + "brotli": [ + "brotli; platform_python_implementation == 'CPython'", + "brotlicffi; platform_python_implementation != 'CPython'", + ] + }, **kwargs ) diff --git a/tornado/web.py b/tornado/web.py index c4f8836770..7ef3b7451c 100644 --- a/tornado/web.py +++ b/tornado/web.py @@ -112,18 +112,19 @@ async def main(): url = URLSpec from typing import ( - Dict, Any, - Union, - Optional, Awaitable, - Tuple, - List, Callable, + Dict, + FrozenSet, Generator, Iterable, - Generator, + List, + Optional, + Set, + Tuple, Type, TypeVar, + Union, cast, overload, ) @@ -2105,8 +2106,10 @@ def __init__( **settings: Any, ) -> None: if transforms is None: - self.transforms = [] # type: List[Type[OutputTransform]] - if settings.get("compress_response") or settings.get("gzip"): + self.transforms = [] # type: List[Callable[[httputil.HTTPServerRequest], OutputTransform]] + if settings.get("compress_response"): + self.transforms.append(CompressingOutputTransform) + elif settings.get("gzip"): self.transforms.append(GZipContentEncoding) else: self.transforms = transforms @@ -3138,34 +3141,19 @@ def transform_chunk(self, chunk: bytes, finishing: bool) -> bytes: return chunk -class GZipContentEncoding(OutputTransform): - """Applies the gzip content encoding to the response. - - See http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.11 - - .. versionchanged:: 4.0 - Now compresses all mime types beginning with ``text/``, instead - of just a whitelist. (the whitelist is still used for certain - non-text mime types). - """ - +class _CompressingOutputTransformABC(OutputTransform): + """ABC for classes that compress outputs.""" # Whitelist of compressible mime types (in addition to any types # beginning with "text/"). - CONTENT_TYPES = set( - [ - "application/javascript", - "application/x-javascript", - "application/xml", - "application/atom+xml", - "application/json", - "application/xhtml+xml", - "image/svg+xml", - ] - ) - # Python's GzipFile defaults to level 9, while most other gzip - # tools (including gzip itself) default to 6, which is probably a - # better CPU/size tradeoff. - GZIP_LEVEL = 6 + CONTENT_TYPES = { + "application/javascript", + "application/x-javascript", + "application/xml", + "application/atom+xml", + "application/json", + "application/xhtml+xml", + "image/svg+xml", + } # Responses that are too short are unlikely to benefit from gzipping # after considering the "Content-Encoding: gzip" header and the header # inside the gzip encoding. @@ -3173,8 +3161,24 @@ class GZipContentEncoding(OutputTransform): # regardless of size. MIN_LENGTH = 1024 + CONTENT_ENCODING: str = None + """The encoding used, in sub-classes it will be equal to CONTENT_ENCODING.""" + + _compressing: bool + """If this is False the output will not be transformed.""" + + @staticmethod + def accepted_encodings(request: httputil.HTTPServerRequest) -> "frozenset[str]": + return frozenset( + map( + str.strip, + request.headers.get("Accept-Encoding", "").lower().split(",") + ) + ) + def __init__(self, request: httputil.HTTPServerRequest) -> None: - self._gzipping = "gzip" in request.headers.get("Accept-Encoding", "") + super().__init__(request) + self._compressing = self.CONTENT_ENCODING in self.accepted_encodings(request) def _compressible_type(self, ctype: str) -> bool: return ctype.startswith("text/") or ctype in self.CONTENT_TYPES @@ -3186,38 +3190,73 @@ def transform_first_chunk( chunk: bytes, finishing: bool, ) -> Tuple[int, httputil.HTTPHeaders, bytes]: - # TODO: can/should this type be inherited from the superclass? if "Vary" in headers: headers["Vary"] += ", Accept-Encoding" else: headers["Vary"] = "Accept-Encoding" - if self._gzipping: - ctype = _unicode(headers.get("Content-Type", "")).split(";")[0] - self._gzipping = ( - self._compressible_type(ctype) - and (not finishing or len(chunk) >= self.MIN_LENGTH) - and ("Content-Encoding" not in headers) - ) - if self._gzipping: - headers["Content-Encoding"] = "gzip" - self._gzip_value = BytesIO() - self._gzip_file = gzip.GzipFile( - mode="w", fileobj=self._gzip_value, compresslevel=self.GZIP_LEVEL - ) - chunk = self.transform_chunk(chunk, finishing) - if "Content-Length" in headers: - # The original content length is no longer correct. - # If this is the last (and only) chunk, we can set the new - # content-length; otherwise we remove it and fall back to - # chunked encoding. - if finishing: - headers["Content-Length"] = str(len(chunk)) - else: - del headers["Content-Length"] + if not self._compressing: + return status_code, headers, chunk + ctype = _unicode(headers.get("Content-Type", "")).split(";")[0] + if not ( + self._compressible_type(ctype) + and (not finishing or len(chunk) >= self.MIN_LENGTH) + and ("Content-Encoding" not in headers) + ): + self._compressing = None + return status_code, headers, chunk + + headers["Content-Encoding"] = self.CONTENT_ENCODING + + self.prepare_transform() + + chunk = self.transform_chunk(chunk, finishing) + if "Content-Length" in headers: + # The original content length is no longer correct. + # If this is the last (and only) chunk, we can set the new + # content-length; otherwise we remove it and fall back to + # chunked encoding. + if finishing: + headers["Content-Length"] = str(len(chunk)) + else: + del headers["Content-Length"] + return status_code, headers, chunk + def prepare_transform(self) -> None: + raise NotImplementedError() + def transform_chunk(self, chunk: bytes, finishing: bool) -> bytes: - if self._gzipping: + raise NotImplementedError() + + +class GZipContentEncoding(_CompressingOutputTransformABC): + """Applies the gzip content encoding to the response. + + See http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.11 + + .. versionchanged:: 4.0 + Now compresses all mime types beginning with ``text/``, instead + of just a whitelist. (the whitelist is still used for certain + non-text mime types). + """ + # Python's GzipFile defaults to level 9, while most other gzip + # tools (including gzip itself) default to 6, which is probably a + # better CPU/size tradeoff. + GZIP_LEVEL = 6 + + CONTENT_ENCODING = "gzip" + + _gzip_value: BytesIO + _gzip_file: gzip.GzipFile + + def prepare_transform(self) -> None: + self._gzip_value = BytesIO() + self._gzip_file = gzip.GzipFile( + mode="w", fileobj=self._gzip_value, compresslevel=self.GZIP_LEVEL + ) + + def transform_chunk(self, chunk: bytes, finishing: bool) -> bytes: + if self._compressing: self._gzip_file.write(chunk) if finishing: self._gzip_file.close() @@ -3229,6 +3268,68 @@ def transform_chunk(self, chunk: bytes, finishing: bool) -> bytes: return chunk +class BrotliContentEncoding(_CompressingOutputTransformABC): + """Compresses the response with brotli.""" + CONTENT_ENCODING = "br" + + # Python's Brotli defaults to level 11 + BROTLI_LEVEL = 8 + + def prepare_transform(self) -> None: + self._compressor = brotli.Compressor( + mode=brotli.MODE_TEXT, quality=self.BROTLI_LEVEL, lgwin=22, lgblock=0 + ) + + def transform_chunk(self, chunk: bytes, finishing: bool) -> bytes: + if self._compressing: + chunk = self._compressor.process(chunk) + if finishing: + chunk += self._compressor.finish() + return chunk + + +try: + try: + import brotlicffi as brotli + except ImportError: + import brotli +except ImportError: + brotli = None +else: + pass + + +COMPRESSION_ALGORITHMS: "List[Tuple[str, Type[_CompressingOutputTransformABC]]]" = [ + *( + [ + ("br", BrotliContentEncoding) + ] + if brotli + else [] + ), + ("gzip", GZipContentEncoding), +] +"""The available compression algorithms sorted by preference.""" + + + +def CompressingOutputTransform(request: httputil.HTTPServerRequest): + """Applies the compression to the response.""" + accepted_encodings = _CompressingOutputTransformABC.accepted_encodings(request) + compressing: "Optional[type[_CompressingOutputTransformABC]]" = None + content_encoding: str = "" + for name, class_ in COMPRESSION_ALGORITHMS: + if name in accepted_encodings: + compressing = class_ + content_encoding = name + break + + if not compressing or not content_encoding: + return OutputTransform(request) + + return compressing(request) + + def authenticated( method: Callable[..., Optional[Awaitable[None]]] ) -> Callable[..., Optional[Awaitable[None]]]: From 0cf5c8ac00664b17266a99504ba7b992c27c90f0 Mon Sep 17 00:00:00 2001 From: Joshix-1 Date: Sun, 19 Feb 2023 22:00:00 +0000 Subject: [PATCH 2/2] fix typing and formatting --- tornado/web.py | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/tornado/web.py b/tornado/web.py index 7ef3b7451c..6b35ae54e8 100644 --- a/tornado/web.py +++ b/tornado/web.py @@ -116,11 +116,12 @@ async def main(): Awaitable, Callable, Dict, - FrozenSet, Generator, + FrozenSet, + Generator, Iterable, List, + MutableSequence, Optional, - Set, Tuple, Type, TypeVar, @@ -2102,15 +2103,19 @@ def __init__( self, handlers: Optional[_RuleList] = None, default_host: Optional[str] = None, - transforms: Optional[List[Type["OutputTransform"]]] = None, + transforms: Optional[ + MutableSequence[Callable[[httputil.HTTPServerRequest], "OutputTransform"]] + ] = None, **settings: Any, ) -> None: if transforms is None: - self.transforms = [] # type: List[Callable[[httputil.HTTPServerRequest], OutputTransform]] + self.transforms = ( + [] + ) # type: MutableSequence[Callable[[httputil.HTTPServerRequest], OutputTransform]] if settings.get("compress_response"): - self.transforms.append(CompressingOutputTransform) + self.transforms = [CompressingOutputTransform] elif settings.get("gzip"): - self.transforms.append(GZipContentEncoding) + self.transforms = [GZipContentEncoding] else: self.transforms = transforms self.default_host = default_host @@ -3143,6 +3148,7 @@ def transform_chunk(self, chunk: bytes, finishing: bool) -> bytes: class _CompressingOutputTransformABC(OutputTransform): """ABC for classes that compress outputs.""" + # Whitelist of compressible mime types (in addition to any types # beginning with "text/"). CONTENT_TYPES = { @@ -3161,18 +3167,17 @@ class _CompressingOutputTransformABC(OutputTransform): # regardless of size. MIN_LENGTH = 1024 - CONTENT_ENCODING: str = None + CONTENT_ENCODING: str = "" """The encoding used, in sub-classes it will be equal to CONTENT_ENCODING.""" _compressing: bool """If this is False the output will not be transformed.""" @staticmethod - def accepted_encodings(request: httputil.HTTPServerRequest) -> "frozenset[str]": + def accepted_encodings(request: httputil.HTTPServerRequest) -> "FrozenSet[str]": return frozenset( map( - str.strip, - request.headers.get("Accept-Encoding", "").lower().split(",") + str.strip, request.headers.get("Accept-Encoding", "").lower().split(",") ) ) @@ -3202,7 +3207,7 @@ def transform_first_chunk( and (not finishing or len(chunk) >= self.MIN_LENGTH) and ("Content-Encoding" not in headers) ): - self._compressing = None + self._compressing = False return status_code, headers, chunk headers["Content-Encoding"] = self.CONTENT_ENCODING @@ -3239,6 +3244,7 @@ class GZipContentEncoding(_CompressingOutputTransformABC): of just a whitelist. (the whitelist is still used for certain non-text mime types). """ + # Python's GzipFile defaults to level 9, while most other gzip # tools (including gzip itself) default to 6, which is probably a # better CPU/size tradeoff. @@ -3270,6 +3276,7 @@ def transform_chunk(self, chunk: bytes, finishing: bool) -> bytes: class BrotliContentEncoding(_CompressingOutputTransformABC): """Compresses the response with brotli.""" + CONTENT_ENCODING = "br" # Python's Brotli defaults to level 11 @@ -3290,9 +3297,9 @@ def transform_chunk(self, chunk: bytes, finishing: bool) -> bytes: try: try: - import brotlicffi as brotli + import brotlicffi as brotli # type: ignore[import] except ImportError: - import brotli + import brotli # type: ignore[import] except ImportError: brotli = None else: @@ -3300,23 +3307,16 @@ def transform_chunk(self, chunk: bytes, finishing: bool) -> bytes: COMPRESSION_ALGORITHMS: "List[Tuple[str, Type[_CompressingOutputTransformABC]]]" = [ - *( - [ - ("br", BrotliContentEncoding) - ] - if brotli - else [] - ), + *([("br", BrotliContentEncoding)] if brotli else []), ("gzip", GZipContentEncoding), ] """The available compression algorithms sorted by preference.""" - -def CompressingOutputTransform(request: httputil.HTTPServerRequest): +def CompressingOutputTransform(request: httputil.HTTPServerRequest) -> OutputTransform: """Applies the compression to the response.""" accepted_encodings = _CompressingOutputTransformABC.accepted_encodings(request) - compressing: "Optional[type[_CompressingOutputTransformABC]]" = None + compressing: "Optional[Type[_CompressingOutputTransformABC]]" = None content_encoding: str = "" for name, class_ in COMPRESSION_ALGORITHMS: if name in accepted_encodings: