From e59f42ea02866aaf9146d610667bcabca76fa419 Mon Sep 17 00:00:00 2001 From: Mea1Ma Date: Mon, 8 Jun 2026 10:15:42 +0800 Subject: [PATCH 1/2] fix(dataset): compare on-the-wire bytes so gzip responses aren't flagged as truncated downloads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The truncation check in url.py compared bytes *written to disk* against Content-Length. That breaks for Content-Encoding responses: when the server sends gzip, Content-Length is the compressed size while iter_bytes() yields the larger decompressed body, so written bytes always exceed Content-Length and the download is falsely rejected. Compare r.num_bytes_downloaded (raw on-the-wire bytes, before httpx decompresses) against Content-Length instead — the two are now measured on the same, compressed scale, so gzip-served files pass while genuinely truncated transfers still fail. Regression: SQuAD's train-v2.0.json is gzip-served; Content-Length=9551051 but the decoded body is ~42MB, which the old written-bytes check rejected. Co-Authored-By: Claude Opus 4.7 (1M context) --- sieval/datasets/downloaders/url.py | 11 +++++--- tests/unit/datasets/downloaders/test_url.py | 31 +++++++++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/sieval/datasets/downloaders/url.py b/sieval/datasets/downloaders/url.py index 8764fd4d..6ba222d7 100644 --- a/sieval/datasets/downloaders/url.py +++ b/sieval/datasets/downloaders/url.py @@ -35,16 +35,19 @@ def download( with httpx.stream("GET", url, timeout=_TIMEOUT, follow_redirects=True) as r: r.raise_for_status() # Catches 2xx responses that dropped the connection mid-stream. + # Compare against on-the-wire bytes, not bytes written: when the + # server sends Content-Encoding (e.g. gzip), Content-Length is the + # compressed size while iter_bytes() yields the larger decompressed + # body, so comparing written bytes would falsely flag truncation. expected = _parse_content_length(r.headers.get("content-length")) - written = 0 with tmp.open("wb") as f: for chunk in r.iter_bytes(chunk_size=1 << 16): f.write(chunk) - written += len(chunk) - if expected is not None and written != expected: + received = r.num_bytes_downloaded + if expected is not None and received != expected: raise RuntimeError( f"truncated download from {url}: " - f"Content-Length={expected} but wrote {written} bytes" + f"Content-Length={expected} but received {received} bytes" ) tmp.replace(target) except BaseException: diff --git a/tests/unit/datasets/downloaders/test_url.py b/tests/unit/datasets/downloaders/test_url.py index 1bcdd3f1..6a2db36f 100644 --- a/tests/unit/datasets/downloaders/test_url.py +++ b/tests/unit/datasets/downloaders/test_url.py @@ -19,6 +19,7 @@ def test_download_uses_per_phase_timeout(tmp_path): mock_resp = MagicMock() mock_resp.iter_bytes.return_value = [b"x"] mock_resp.raise_for_status.return_value = None + mock_resp.num_bytes_downloaded = 1 mock_stream.return_value.__enter__.return_value = mock_resp h.download("url:https://example.com/foo.csv", tmp_path, "foo", force=False) timeout_arg = mock_stream.call_args.kwargs.get("timeout") @@ -44,6 +45,7 @@ def test_download_writes_file_at_basename(tmp_path): mock_resp.iter_bytes.return_value = [b"hello"] mock_resp.raise_for_status.return_value = None mock_resp.headers = {"content-length": "5"} + mock_resp.num_bytes_downloaded = 5 mock_stream.return_value.__enter__.return_value = mock_resp h.download( "url:https://example.com/foo.csv", @@ -81,6 +83,7 @@ def test_download_force_redownloads(tmp_path): mock_resp.iter_bytes.return_value = [b"fresh"] mock_resp.raise_for_status.return_value = None mock_resp.headers = {"content-length": "5"} + mock_resp.num_bytes_downloaded = 5 mock_stream.return_value.__enter__.return_value = mock_resp h.download( "url:https://example.com/foo.csv", @@ -103,6 +106,7 @@ def test_download_rejects_truncated_stream(tmp_path): mock_resp.iter_bytes.return_value = [b"abc"] mock_resp.raise_for_status.return_value = None mock_resp.headers = {"content-length": "100"} + mock_resp.num_bytes_downloaded = 3 # connection died after 3 raw bytes mock_stream.return_value.__enter__.return_value = mock_resp with pytest.raises(RuntimeError, match="truncated download"): h.download( @@ -116,6 +120,33 @@ def test_download_rejects_truncated_stream(tmp_path): assert not (tmp_path / "foo" / "foo.csv.partial").exists() +def test_download_accepts_compressed_response(tmp_path): + """Server sends Content-Encoding (e.g. gzip): Content-Length is the + compressed size while iter_bytes() yields the larger decompressed body. + The check must compare on-the-wire bytes (num_bytes_downloaded) against + Content-Length, not the decompressed bytes written — otherwise every + compressed download falsely trips the truncation guard. + + Regression: SQuAD's train-v2.0.json is gzip-served; Content-Length=9551051 + but the decoded body is ~42MB, which the old written-bytes check rejected.""" + h = URLHandler() + with patch("sieval.datasets.downloaders.url.httpx.stream") as mock_stream: + mock_resp = MagicMock() + # Decompressed body is far larger than the compressed Content-Length. + mock_resp.iter_bytes.return_value = [b"x" * 42, b"y" * 42] + mock_resp.raise_for_status.return_value = None + mock_resp.headers = {"content-length": "9", "content-encoding": "gzip"} + mock_resp.num_bytes_downloaded = 9 # raw compressed bytes on the wire + mock_stream.return_value.__enter__.return_value = mock_resp + h.download( + "url:https://example.com/foo.json", + dest_root=tmp_path, + dataset_name="foo", + force=False, + ) + assert (tmp_path / "foo" / "foo.json").read_bytes() == b"x" * 42 + b"y" * 42 + + def test_download_accepts_missing_content_length(tmp_path): """Chunked transfer-encoded responses often omit Content-Length. The truncation check must degrade gracefully rather than reject every From 7b969040cd78c44c0e8ece827502dbd91fd34c71 Mon Sep 17 00:00:00 2001 From: Ethan Date: Mon, 29 Jun 2026 16:38:59 +0800 Subject: [PATCH 2/2] fix(dataset): reword url size-mismatch error; pin gzip test to RULER's dev-v2.0.json source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The byte-count guard rejects any num_bytes_downloaded != Content-Length mismatch, including an over-read, so "truncated download" was a misnomer for that direction — reword to "size mismatch". Also align test_download_accepts_compressed_response's docstring to the file RULER actually fetches (dev-v2.0.json: Content-Length=800683, decoded ~4.4MB) instead of train-v2.0.json, so the Regression note describes the real download path. Co-Authored-By: Claude Opus 4.8 (1M context) --- sieval/datasets/downloaders/url.py | 2 +- tests/unit/datasets/downloaders/test_url.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/sieval/datasets/downloaders/url.py b/sieval/datasets/downloaders/url.py index 6ba222d7..db1b40a1 100644 --- a/sieval/datasets/downloaders/url.py +++ b/sieval/datasets/downloaders/url.py @@ -46,7 +46,7 @@ def download( received = r.num_bytes_downloaded if expected is not None and received != expected: raise RuntimeError( - f"truncated download from {url}: " + f"size mismatch on download from {url}: " f"Content-Length={expected} but received {received} bytes" ) tmp.replace(target) diff --git a/tests/unit/datasets/downloaders/test_url.py b/tests/unit/datasets/downloaders/test_url.py index 6a2db36f..3593b47b 100644 --- a/tests/unit/datasets/downloaders/test_url.py +++ b/tests/unit/datasets/downloaders/test_url.py @@ -108,7 +108,7 @@ def test_download_rejects_truncated_stream(tmp_path): mock_resp.headers = {"content-length": "100"} mock_resp.num_bytes_downloaded = 3 # connection died after 3 raw bytes mock_stream.return_value.__enter__.return_value = mock_resp - with pytest.raises(RuntimeError, match="truncated download"): + with pytest.raises(RuntimeError, match="size mismatch"): h.download( "url:https://example.com/foo.csv", dest_root=tmp_path, @@ -127,8 +127,9 @@ def test_download_accepts_compressed_response(tmp_path): Content-Length, not the decompressed bytes written — otherwise every compressed download falsely trips the truncation guard. - Regression: SQuAD's train-v2.0.json is gzip-served; Content-Length=9551051 - but the decoded body is ~42MB, which the old written-bytes check rejected.""" + Regression: RULER's SQuAD source (dev-v2.0.json) is gzip-served; + Content-Length=800683 but the decoded body is ~4.4MB, which the old + written-bytes check rejected.""" h = URLHandler() with patch("sieval.datasets.downloaders.url.httpx.stream") as mock_stream: mock_resp = MagicMock()