Skip to content

Commit 7b6c9a0

Browse files
committed
Fix: Correctly calculate starting offset for retries of ranged reads (#1376)
1 parent 4d20a8e commit 7b6c9a0

File tree

2 files changed

+74
-2
lines changed

2 files changed

+74
-2
lines changed

google/cloud/storage/_media/requests/download.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -194,7 +194,7 @@ def retriable_request():
194194
# received using a range request, and set object generation query param.
195195
if self._bytes_downloaded > 0:
196196
_download.add_bytes_range(
197-
self._bytes_downloaded, self.end, self._headers
197+
(self.start or 0) + self._bytes_downloaded, self.end, self._headers
198198
)
199199
request_kwargs["headers"] = self._headers
200200

@@ -382,7 +382,7 @@ def retriable_request():
382382
# received using a range request, and set object generation query param.
383383
if self._bytes_downloaded > 0:
384384
_download.add_bytes_range(
385-
self._bytes_downloaded, self.end, self._headers
385+
(self.start or 0) + self._bytes_downloaded, self.end, self._headers
386386
)
387387
request_kwargs["headers"] = self._headers
388388

tests/resumable_media/unit/requests/test_download.py

+72
Original file line numberDiff line numberDiff line change
@@ -443,6 +443,42 @@ def test_consume_w_bytes_downloaded(self):
443443
range_bytes = "bytes={:d}-{:d}".format(offset, end)
444444
assert download._headers["range"] == range_bytes
445445

446+
def test_consume_w_bytes_downloaded_range_read(self):
447+
stream = io.BytesIO()
448+
chunks = (b"up down ", b"charlie ", b"brown")
449+
start = 1024
450+
end = 65536
451+
452+
download = download_mod.Download(
453+
EXAMPLE_URL,
454+
stream=stream,
455+
start=start,
456+
end=end,
457+
headers=None,
458+
checksum="md5",
459+
)
460+
transport = mock.Mock(spec=["request"])
461+
transport.request.return_value = _mock_response(chunks=chunks, headers=None)
462+
463+
assert download._bytes_downloaded == 0
464+
465+
# Mock a retry operation with bytes already downloaded in the stream and checksum stored
466+
offset = 256
467+
download._bytes_downloaded = offset
468+
download._expected_checksum = None
469+
download._checksum_object = _helpers._DoNothingHash()
470+
download.consume(transport)
471+
472+
called_kwargs = {
473+
"data": None,
474+
"headers": download._headers,
475+
"timeout": EXPECTED_TIMEOUT,
476+
"stream": True,
477+
}
478+
transport.request.assert_called_once_with("GET", EXAMPLE_URL, **called_kwargs)
479+
range_bytes = "bytes={:d}-{:d}".format(offset + start, end)
480+
assert download._headers["range"] == range_bytes
481+
446482
def test_consume_gzip_reset_stream_w_bytes_downloaded(self):
447483
stream = io.BytesIO()
448484
chunks = (b"up down ", b"charlie ", b"brown")
@@ -859,6 +895,42 @@ def test_consume_w_bytes_downloaded(self):
859895
range_bytes = "bytes={:d}-{:d}".format(offset, end)
860896
assert download._headers["range"] == range_bytes
861897

898+
def test_consume_w_bytes_downloaded_range_read(self):
899+
stream = io.BytesIO()
900+
chunks = (b"up down ", b"charlie ", b"brown")
901+
start = 1024
902+
end = 65536
903+
904+
download = download_mod.RawDownload(
905+
EXAMPLE_URL,
906+
stream=stream,
907+
start=start,
908+
end=end,
909+
headers=None,
910+
checksum="md5",
911+
)
912+
transport = mock.Mock(spec=["request"])
913+
transport.request.return_value = _mock_raw_response(chunks=chunks, headers=None)
914+
915+
assert download._bytes_downloaded == 0
916+
917+
# Mock a retry operation with bytes already downloaded in the stream and checksum stored
918+
offset = 256
919+
download._bytes_downloaded = offset
920+
download._expected_checksum = None
921+
download._checksum_object = _helpers._DoNothingHash()
922+
download.consume(transport)
923+
924+
called_kwargs = {
925+
"data": None,
926+
"headers": download._headers,
927+
"timeout": EXPECTED_TIMEOUT,
928+
"stream": True,
929+
}
930+
transport.request.assert_called_once_with("GET", EXAMPLE_URL, **called_kwargs)
931+
range_bytes = "bytes={:d}-{:d}".format(start + offset, end)
932+
assert download._headers["range"] == range_bytes
933+
862934
def test_consume_gzip_reset_stream_w_bytes_downloaded(self):
863935
stream = io.BytesIO()
864936
chunks = (b"up down ", b"charlie ", b"brown")

0 commit comments

Comments
 (0)