diff --git a/README.md b/README.md index 2ccecd578c..96042e4c41 100644 --- a/README.md +++ b/README.md @@ -136,7 +136,7 @@ As well as these optional installs: * `rich` - Rich terminal support. *(Optional, with `httpx[cli]`)* * `click` - Command line client support. *(Optional, with `httpx[cli]`)* * `brotli` or `brotlicffi` - Decoding for "brotli" compressed responses. *(Optional, with `httpx[brotli]`)* -* `zstandard` - Decoding for "zstd" compressed responses. *(Optional, with `httpx[zstd]`)* +* `backports.zstd` - Decoding for "zstd" compressed responses on Python before 3.14. *(Optional, with `httpx[zstd]`)* A huge amount of credit is due to `requests` for the API layout that much of this work follows, as well as to `urllib3` for plenty of design diff --git a/docs/quickstart.md b/docs/quickstart.md index e140b53cd7..49e2eb4de3 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -100,8 +100,8 @@ b'\n\n\nExample Domain...' Any `gzip` and `deflate` HTTP response encodings will automatically be decoded for you. If `brotlipy` is installed, then the `brotli` response -encoding will be supported. If `zstandard` is installed, then `zstd` -response encodings will also be supported. +encoding will be supported. If the Python version used is 3.14 or higher or +if `backports.zstd` is installed, then `zstd` response encodings will also be supported. For example, to create an image from binary data returned by a request, you can use the following code: diff --git a/httpx/_decoders.py b/httpx/_decoders.py index 899dfada87..9a89857760 100644 --- a/httpx/_decoders.py +++ b/httpx/_decoders.py @@ -8,6 +8,7 @@ import codecs import io +import sys import typing import zlib @@ -28,9 +29,12 @@ # Zstandard support is optional try: - import zstandard + if sys.version_info >= (3, 14): + from compression import zstd # pragma: no cover + else: + from backports import zstd # pragma: no cover except ImportError: # pragma: no cover - zstandard = None # type: ignore + zstd = None # type: ignore class ContentDecoder: @@ -162,42 +166,40 @@ class ZStandardDecoder(ContentDecoder): """ Handle 'zstd' RFC 8878 decoding. - Requires `pip install zstandard`. + Requires `pip install backports.zstd` for Python before 3.14. Can be installed as a dependency of httpx using `pip install httpx[zstd]`. """ # inspired by the ZstdDecoder implementation in urllib3 def __init__(self) -> None: - if zstandard is None: # pragma: no cover + if zstd is None: # pragma: no cover raise ImportError( "Using 'ZStandardDecoder', ..." "Make sure to install httpx using `pip install httpx[zstd]`." ) from None - self.decompressor = zstandard.ZstdDecompressor().decompressobj() - self.seen_data = False + self.decompressor = zstd.ZstdDecompressor() + self.at_valid_eof = True def decode(self, data: bytes) -> bytes: - assert zstandard is not None - self.seen_data = True + assert zstd is not None output = io.BytesIO() try: - output.write(self.decompressor.decompress(data)) - while self.decompressor.eof and self.decompressor.unused_data: - unused_data = self.decompressor.unused_data - self.decompressor = zstandard.ZstdDecompressor().decompressobj() - output.write(self.decompressor.decompress(unused_data)) - except zstandard.ZstdError as exc: + self.at_valid_eof = False + while data: + output.write(self.decompressor.decompress(data)) + data = self.decompressor.unused_data + if self.decompressor.eof: + self.decompressor = zstd.ZstdDecompressor() + self.at_valid_eof = not data + except zstd.ZstdError as exc: raise DecodingError(str(exc)) from exc return output.getvalue() def flush(self) -> bytes: - if not self.seen_data: + if self.at_valid_eof: return b"" - ret = self.decompressor.flush() # note: this is a no-op - if not self.decompressor.eof: - raise DecodingError("Zstandard data is incomplete") # pragma: no cover - return bytes(ret) + raise DecodingError("Zstandard data is incomplete") # pragma: no cover class MultiDecoder(ContentDecoder): @@ -389,5 +391,5 @@ def flush(self) -> list[str]: if brotli is None: SUPPORTED_DECODERS.pop("br") # pragma: no cover -if zstandard is None: +if zstd is None: SUPPORTED_DECODERS.pop("zstd") # pragma: no cover diff --git a/pyproject.toml b/pyproject.toml index fc3e95ea74..48bd19629e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ socks = [ "socksio==1.*", ] zstd = [ - "zstandard>=0.18.0", + "backports.zstd>=1.0.0 ; python_version < '3.14'", ] [project.scripts] diff --git a/tests/test_decoders.py b/tests/test_decoders.py index 9ffaba189d..4c14bb52c8 100644 --- a/tests/test_decoders.py +++ b/tests/test_decoders.py @@ -1,15 +1,20 @@ from __future__ import annotations import io +import sys import typing import zlib import chardet import pytest -import zstandard as zstd import httpx +if sys.version_info >= (3, 14): + from compression import zstd # pragma: no cover +else: + from backports import zstd # pragma: no cover + def test_deflate(): """ @@ -115,7 +120,7 @@ def test_zstd_truncated(): httpx.Response( 200, headers=headers, - content=compressed_body[1:3], + content=compressed_body[:-1], ) @@ -141,6 +146,39 @@ def test_zstd_multiframe(): assert response.content == b"foobar" +def test_zstd_truncated_multiframe(): + body = b"test 123" + compressed_body = zstd.compress(body) + + headers = [(b"Content-Encoding", b"zstd")] + with pytest.raises(httpx.DecodingError): + httpx.Response( + 200, + headers=headers, + content=compressed_body + compressed_body[:-1], + ) + + +def test_zstd_streaming_multiple_frames(): + body1 = b"test 123 " + body2 = b"another frame" + + # Create two separate complete frames + frame1 = zstd.compress(body1) + frame2 = zstd.compress(body2) + + # Create an iterator that yields frames separately + def content_iterator() -> typing.Iterator[bytes]: + yield frame1 + yield frame2 + + headers = [(b"Content-Encoding", b"zstd")] + response = httpx.Response(200, headers=headers, content=content_iterator()) + response.read() + + assert response.content == body1 + body2 + + def test_multi(): body = b"test 123"