From d7265fa3ee2a8a3df9d89fe5253b3f4de22246c1 Mon Sep 17 00:00:00 2001 From: Felipe Fumero Date: Sat, 21 Nov 2026 16:53:51 -0600 Subject: [PATCH] fix: harden multipart and unbound transfers --- b2sdk/_internal/transfer/emerge/executor.py | 69 ++++++++++++++----- .../transfer/emerge/unbound_write_intent.py | 47 +++++++++++-- .../transfer/outbound/upload_manager.py | 24 ++++--- changelog.d/+transfer-audit.fixed.md | 1 + test/unit/bucket/test_regression_bucket.py | 9 ++- .../internal/test_emerge_cancel_siblings.py | 48 +++++++++++++ .../internal/test_unbound_write_intent.py | 15 ++++ .../test_upload_checksum_optimized.py | 60 ++++++++++++++++ test/unit/stream/test_regression_streams.py | 5 -- 9 files changed, 232 insertions(+), 46 deletions(-) create mode 100644 changelog.d/+transfer-audit.fixed.md create mode 100644 test/unit/internal/test_emerge_cancel_siblings.py create mode 100644 test/unit/internal/test_upload_checksum_optimized.py diff --git a/b2sdk/_internal/transfer/emerge/executor.py b/b2sdk/_internal/transfer/emerge/executor.py index 779cd5bd3..35369bed3 100644 --- a/b2sdk/_internal/transfer/emerge/executor.py +++ b/b2sdk/_internal/transfer/emerge/executor.py @@ -212,25 +212,56 @@ def execute_plan(self, emerge_plan: StreamingEmergePlan): large_file_upload_state = LargeFileUploadState(self.progress_listener) part_futures = [] - for part_number, emerge_part in emerge_plan.enumerate_emerge_parts(): - execution_step_factory = LargeFileEmergeExecutionStepFactory( - self, - emerge_part, - part_number, - file_id, - large_file_upload_state, - finished_parts=finished_parts, - # it already knows encryption from BaseMergeExecution being passed as self - ) - execution_step = execution_step_factory.get_execution_step() - future = self._execute_step(execution_step) - part_futures.append(future) - - # Collect the sha1 checksums of the parts as the uploads finish. - # If any of them raised an exception, that same exception will - # be raised here by result() - part_sha1_array = [f.result()['contentSha1'] for f in part_futures] - + failure_lock = threading.RLock() + first_failure = [] + + def cancel_on_failure(future): + if future.cancelled(): + return + error = future.exception() + if error is None: + return + with failure_lock: + if not first_failure: + first_failure.append(error) + large_file_upload_state.set_error(str(error)) + for sibling in part_futures: + if sibling is not future: + sibling.cancel() + + try: + for part_number, emerge_part in emerge_plan.enumerate_emerge_parts(): + with failure_lock: + if first_failure: + raise first_failure[0] + execution_step_factory = LargeFileEmergeExecutionStepFactory( + self, + emerge_part, + part_number, + file_id, + large_file_upload_state, + finished_parts=finished_parts, + ) + execution_step = execution_step_factory.get_execution_step() + future = self._execute_step(execution_step) + with failure_lock: + part_futures.append(future) + future.add_done_callback(cancel_on_failure) + if first_failure: + future.cancel() + raise first_failure[0] + + # Keep the part checksums in part-number order. + part_sha1_array = [f.result()['contentSha1'] for f in part_futures] + except BaseException: + with failure_lock: + for future in part_futures: + future.cancel() + if first_failure: + raise first_failure[0] + raise + + # Leave unfinished uploads on the server after failure so they can be resumed. # Finish the large file response = self.services.session.finish_large_file(file_id, part_sha1_array) return self.services.api.file_version_factory.from_api_response(response) diff --git a/b2sdk/_internal/transfer/emerge/unbound_write_intent.py b/b2sdk/_internal/transfer/emerge/unbound_write_intent.py index 68f1e9fd2..d4fa4b5c3 100644 --- a/b2sdk/_internal/transfer/emerge/unbound_write_intent.py +++ b/b2sdk/_internal/transfer/emerge/unbound_write_intent.py @@ -19,9 +19,9 @@ from b2sdk._internal.transfer.outbound.upload_source import AbstractUploadSource -class IOWrapper(io.BytesIO): +class IOWrapper(io.RawIOBase): """ - Wrapper for BytesIO that knows when it has been read in full. + Seekable, read-only view over an upload buffer. Note that this stream should go through ``emerge_unbound``, as it's the only one that skips ``_get_emerge_parts`` and pushes buffers to the cloud @@ -38,7 +38,7 @@ def __init__( release_function: Callable[[], None], ): """ - Prepares a new ``io.BytesIO`` structure that will call + Prepares a stream over the buffer that will call a ``release_function`` when buffer is read in full. ``release_function`` can be called from another thread. @@ -48,13 +48,48 @@ def __init__( :param data: data to be provided as a stream :param release_function: function to be called when resource will be released """ - super().__init__(data) + super().__init__() + self._view = memoryview(data) + self._position = 0 self.release_function = release_function + def readable(self): + return True + + def seekable(self): + return True + + def tell(self): + self._checkClosed() + return self._position + + def seek(self, offset, whence=io.SEEK_SET): + self._checkClosed() + if whence == io.SEEK_CUR: + offset += self._position + elif whence == io.SEEK_END: + offset += len(self._view) + elif whence != io.SEEK_SET: + raise ValueError('invalid whence') + if offset < 0: + raise ValueError('negative seek position') + self._position = offset + return offset + + def readinto(self, buffer): + self._checkClosed() + count = min(len(buffer), max(0, len(self._view) - self._position)) + buffer[:count] = self._view[self._position : self._position + count] + self._position += count + return count + def close(self): if not self.closed: - self.release_function() - return super().close() + self._view.release() + try: + self.release_function() + finally: + super().close() class UnboundSourceBytes(AbstractUploadSource): diff --git a/b2sdk/_internal/transfer/outbound/upload_manager.py b/b2sdk/_internal/transfer/outbound/upload_manager.py index 842d8c3f0..ff6ad932d 100644 --- a/b2sdk/_internal/transfer/outbound/upload_manager.py +++ b/b2sdk/_internal/transfer/outbound/upload_manager.py @@ -17,6 +17,7 @@ from b2sdk._internal.exception import ( AlreadyFailed, B2Error, + ChecksumMismatch, MaxRetriesExceeded, ) from b2sdk._internal.file_lock import FileRetentionSetting, LegalHold @@ -175,15 +176,17 @@ def close_stream_callback(stream): input_stream, server_side_encryption=encryption, # todo: client side encryption ) - if content_sha1 == HEX_DIGITS_AT_END: - content_sha1 = input_stream.hash - assert content_sha1 == response['contentSha1'] - return response except B2Error as e: if not e.should_retry_upload(): raise exception_list.append(e) self.account_info.clear_bucket_upload_data(bucket_id) + else: + if content_sha1 == HEX_DIGITS_AT_END: + content_sha1 = input_stream.hash + if content_sha1 != response['contentSha1']: + raise ChecksumMismatch('sha1', content_sha1, response['contentSha1']) + return response large_file_upload_state.set_error(str(exception_list[-1])) raise MaxRetriesExceeded(self.MAX_UPLOAD_ATTEMPTS, exception_list) @@ -228,17 +231,16 @@ def _upload_small_file( legal_hold=legal_hold, custom_upload_timestamp=custom_upload_timestamp, ) - if content_sha1 == HEX_DIGITS_AT_END: - content_sha1 = input_stream.hash - assert ( - content_sha1 == 'do_not_verify' or content_sha1 == response['contentSha1'] - ), '{} != {}'.format(content_sha1, response['contentSha1']) - return self.services.api.file_version_factory.from_api_response(response) - except B2Error as e: if not e.should_retry_upload(): raise exception_info_list.append(e) self.account_info.clear_bucket_upload_data(bucket_id) + else: + if content_sha1 == HEX_DIGITS_AT_END: + content_sha1 = input_stream.hash + if content_sha1 != 'do_not_verify' and content_sha1 != response['contentSha1']: + raise ChecksumMismatch('sha1', content_sha1, response['contentSha1']) + return self.services.api.file_version_factory.from_api_response(response) raise MaxRetriesExceeded(self.MAX_UPLOAD_ATTEMPTS, exception_info_list) diff --git a/changelog.d/+transfer-audit.fixed.md b/changelog.d/+transfer-audit.fixed.md new file mode 100644 index 000000000..4d68c0079 --- /dev/null +++ b/changelog.d/+transfer-audit.fixed.md @@ -0,0 +1 @@ +Reject incorrect upload checksums under Python optimization, cancel pending multipart work after a part fails, and avoid copying unbound upload buffers. diff --git a/test/unit/bucket/test_regression_bucket.py b/test/unit/bucket/test_regression_bucket.py index 4df8b149a..bebbebb68 100644 --- a/test/unit/bucket/test_regression_bucket.py +++ b/test/unit/bucket/test_regression_bucket.py @@ -402,11 +402,6 @@ def _check_iterable_rules(self, input_kind): class TestRegressionLargeUploadFailure: - @known_defect( - 'C8', - 'part uploads keep running after one part of a large upload failed', - raises=DefectStillPresent, - ) def test_c8_failed_part_stops_remaining_part_uploads(self): """C8: once one part fails the whole upload fails; queued parts must not keep uploading.""" from unittest import mock @@ -427,12 +422,14 @@ def test_c8_failed_part_stops_remaining_part_uploads(self): bucket = api.create_bucket('big', 'allPrivate') original = simulator.upload_part part_failed = threading.Event() + queued_sibling = threading.Event() started_after_failure = [] def flaky_upload_part(upload_url, token, part_number, *args, **kwargs): if part_failed.is_set(): started_after_failure.append(part_number) elif part_number == 1: + assert queued_sibling.wait(5), 'a sibling was not queued' part_failed.set() # before raising, so that no later start can be missed raise AccessDenied() return original(upload_url, token, part_number, *args, **kwargs) @@ -445,6 +442,8 @@ def flaky_upload_part(upload_url, token, part_number, *args, **kwargs): def recording_submit(*args, **kwargs): future = real_submit(*args, **kwargs) futures.append(future) + if len(futures) > 1: + queued_sibling.set() return future data = b'x' * (simulator.MIN_PART_SIZE * 6) diff --git a/test/unit/internal/test_emerge_cancel_siblings.py b/test/unit/internal/test_emerge_cancel_siblings.py new file mode 100644 index 000000000..11c2f040f --- /dev/null +++ b/test/unit/internal/test_emerge_cancel_siblings.py @@ -0,0 +1,48 @@ +###################################################################### +# +# File: test/unit/internal/test_emerge_cancel_siblings.py +# +# Copyright 2026 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +from concurrent.futures import Future +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest + +from b2sdk._internal.transfer.emerge.executor import LargeFileEmergeExecution + + +def test_failed_part_cancels_pending_siblings_and_keeps_unfinished_upload(): + services = MagicMock() + execution = LargeFileEmergeExecution(services, 'bucket', 'name', None, {}, MagicMock()) + execution._get_unfinished_file_and_parts = MagicMock( + return_value=(SimpleNamespace(file_id='unfinished'), {}) + ) + plan = MagicMock() + plan.get_total_length.return_value = None + plan.get_plan_id.return_value = None + plan.enumerate_emerge_parts.return_value = iter([(1, object()), (2, object()), (3, object())]) + first, failed, pending = Future(), Future(), Future() + first.set_result({'contentSha1': 'one'}) + steps = [MagicMock(), MagicMock(), MagicMock()] + for step, future in zip(steps, (first, failed, pending)): + step.execute.return_value = future + steps[2].execute.side_effect = lambda: ( + failed.set_exception(ValueError('part failed')), + pending, + )[1] + with patch( + 'b2sdk._internal.transfer.emerge.executor.LargeFileEmergeExecutionStepFactory' + ) as factory: + factory.side_effect = [ + SimpleNamespace(get_execution_step=lambda step=step: step) for step in steps + ] + with pytest.raises(ValueError, match='part failed'): + execution.execute_plan(plan) + assert pending.cancelled() + services.session.finish_large_file.assert_not_called() + services.large_file.cancel_large_file.assert_not_called() diff --git a/test/unit/internal/test_unbound_write_intent.py b/test/unit/internal/test_unbound_write_intent.py index 7be6ce977..1f12ef639 100644 --- a/test/unit/internal/test_unbound_write_intent.py +++ b/test/unit/internal/test_unbound_write_intent.py @@ -31,6 +31,21 @@ def setUp(self) -> None: self.mock_fun = MagicMock() self.wrapper = IOWrapper(self.data, release_function=self.mock_fun) + def test_read_seek_retry_and_release_buffer(self): + data = bytearray(b'abcdef') + release = MagicMock() + stream = IOWrapper(data, release) + self.assertEqual(stream.read(2), b'ab') + self.assertEqual(stream.tell(), 2) + self.assertEqual(stream.seek(-2, io.SEEK_END), 4) + self.assertEqual(stream.read(), b'ef') + self.assertEqual(stream.seek(0), 0) + self.assertEqual(stream.read(), b'abcdef') + stream.close() + stream.close() + release.assert_called_once() + data.extend(b'g') # closing releases the memoryview's export + def test_function_called_on_close_manual(self): self.mock_fun.assert_not_called() diff --git a/test/unit/internal/test_upload_checksum_optimized.py b/test/unit/internal/test_upload_checksum_optimized.py new file mode 100644 index 000000000..e49ba6841 --- /dev/null +++ b/test/unit/internal/test_upload_checksum_optimized.py @@ -0,0 +1,60 @@ +###################################################################### +# +# File: test/unit/internal/test_upload_checksum_optimized.py +# +# Copyright 2026 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### +from __future__ import annotations + +import subprocess +import sys + +import pytest + + +@pytest.mark.parametrize('kind', ['small', 'part']) +@pytest.mark.parametrize('known_sha1', [True, False]) +def test_upload_rejects_bad_server_sha1_under_optimization(kind, known_sha1): + program = """ +from unittest.mock import MagicMock +from b2sdk._internal.exception import ChecksumMismatch +from b2sdk._internal.transfer.outbound.upload_manager import UploadManager +from b2sdk._internal.transfer.outbound.upload_source import UploadSourceBytes +from b2sdk._internal.progress import DoNothingProgressListener +from b2sdk._internal.transfer.outbound.large_file_upload_state import LargeFileUploadState + +services = MagicMock() +def upload_file(*args, **kwargs): + args[6].read() + return {'contentSha1': 'wrong'} +def upload_part(*args, **kwargs): + args[4].read() + return {'contentSha1': 'wrong'} +services.session.upload_file.side_effect = upload_file +services.session.upload_part.side_effect = upload_part +manager = UploadManager(services) +source = UploadSourceBytes(b'hello') +if KNOWN_SHA1: + source.get_content_sha1() +progress = DoNothingProgressListener() +try: + if KIND == 'small': + manager._upload_small_file('bucket', source, 'file', 'text/plain', {}, progress) + else: + manager._upload_part('bucket', 'id', source, 1, LargeFileUploadState(progress), {}, None) +except ChecksumMismatch as error: + if error.expected != source.get_content_sha1() or error.actual != 'wrong': + raise RuntimeError('checksum error has incorrect values') +else: + raise RuntimeError('server checksum mismatch was accepted') +""" + result = subprocess.run( + [sys.executable, '-O', '-c', f'KIND = {kind!r}\nKNOWN_SHA1 = {known_sha1!r}\n' + program], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr diff --git a/test/unit/stream/test_regression_streams.py b/test/unit/stream/test_regression_streams.py index bdffe4a01..24e953325 100644 --- a/test/unit/stream/test_regression_streams.py +++ b/test/unit/stream/test_regression_streams.py @@ -159,11 +159,6 @@ def test_c52_invalid_offset_is_rejected(self, tmp_path, offset): class TestUnboundWriteIntentMemory: - @known_defect( - 'C57', - 'UnboundSourceBytes copies the chunk, using about twice the documented memory', - raises=DefectStillPresent, - ) def test_c57_unbound_buffer_wrapping_does_not_copy_the_chunk(self): """C57: wrapping a bytearray chunk for upload must not allocate a second full copy.""" tracemalloc = pytest.importorskip(