From dd088f85b6b661c6593863b9188d36305fb51ff6 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Fri, 9 Oct 2026 18:00:26 +0200 Subject: [PATCH 01/30] Add conditional batch removal engine Signed-off-by: Konstantin Morozov --- .../ContentAddressed/Backend/CasBackend.h | 22 ++++++++ .../Backend/CasInMemoryBackend.cpp | 14 +++++ .../Backend/CasInMemoryBackend.h | 3 ++ .../Backend/CasInstrumentedBackend.cpp | 9 ++++ .../Backend/CasInstrumentedBackend.h | 3 ++ .../ContentAddressed/Backend/CasRequests.cpp | 53 +++++++++++++++++++ .../ContentAddressed/Backend/CasRequests.h | 7 +++ .../Backend/CasThrottlingBackend.h | 9 ++++ .../tests/gtest_cas_bulk_delete_backend.cpp | 20 +++++++ .../tests/gtest_cas_bulk_delete_engine.cpp | 50 +++++++++++++++++ 10 files changed, 190 insertions(+) diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasBackend.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasBackend.h index eb502f9724c21..d6d24c23c4f1a 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasBackend.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasBackend.h @@ -20,6 +20,11 @@ #include #include +namespace DB::ErrorCodes +{ + extern const int NOT_IMPLEMENTED; +} + namespace DB::Cas { @@ -169,6 +174,11 @@ class Backend /// `DeleteMarker` is a removal that did NOT reclaim: a versioned bucket archived a noncurrent /// version instead. Distinct from `Removed` because reclaiming the storage is the point. enum class RawRemoval : uint8_t { Removed, Gone, Mismatch, DeleteMarker }; + struct RawConditionalRemove + { + String key; + String expected_token; + }; /// Reads the whole object, or nullopt when the key is absent. virtual std::optional read (const String & key, TransportAccess &) = 0; @@ -201,10 +211,22 @@ class Backend /// absence is success. virtual void removeManyWriteOnce(const std::vector & keys, TransportAccess &) = 0; + virtual std::vector removeManyByConditional( + const std::vector & removals, TransportAccess & access) + { + if (removals.size() > 1) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Conditional batch removal is not implemented by this backend"); + if (removals.empty()) + return {}; + return {remove(removals.front().key, removals.front().expected_token, access)}; + } + /// The most keys one `removeManyWriteOnce` sends to the storage as one request; at least 1. A decorator /// must forward it: the default of 1 would turn every batch below it into one-key requests. virtual size_t bulkDeleteKeyLimit() const { return 1; } + virtual size_t conditionalBatchDeleteKeyLimit() const { return 1; } + /// Authoritative, cache-bypassing probe of one key -- see `ProbeOutcome`. DEFAULT (used by every /// backend without sharper raw-error evidence, e.g. `InMemoryBackend`): derived from `head`/`read` /// alone, so it can only distinguish `Present` from `KeyAbsent`, and ANY exception from either diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInMemoryBackend.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInMemoryBackend.cpp index b6bf981a40f1a..d996077166732 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInMemoryBackend.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInMemoryBackend.cpp @@ -326,6 +326,20 @@ void InMemoryBackend::removeManyWriteOnce(const std::vector & keys } } +std::vector InMemoryBackend::removeManyByConditional( + const std::vector & removals, TransportAccess &) +{ + for (const RawConditionalRemove & removal : removals) + checkExpectedValue(removal.key, removal.expected_token); + + std::lock_guard lock(mutex_); + std::vector results; + results.reserve(removals.size()); + for (const RawConditionalRemove & removal : removals) + results.push_back(applyDelete(removal.key, removal.expected_token)); + return results; +} + void InMemoryBackend::failNextBulkRemoveWith(std::exception_ptr error) { std::lock_guard lock(mutex_); diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInMemoryBackend.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInMemoryBackend.h index 9e3100efdf3e9..51f0ae66bedb9 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInMemoryBackend.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInMemoryBackend.h @@ -51,6 +51,8 @@ class InMemoryBackend : public Backend /// `hold_deletes_` exactly as `remove` does: a held delete is queued and lands only on /// `landPendingDelete`. void removeManyWriteOnce(const std::vector & keys, TransportAccess & access) override; + std::vector removeManyByConditional( + const std::vector & removals, TransportAccess & access) override; /// The next `removeManyWriteOnce` throws `error` instead of deleting anything; one-shot, like /// `failNextWriteWith`. void failNextBulkRemoveWith(std::exception_ptr error); @@ -62,6 +64,7 @@ class InMemoryBackend : public Backend /// The emulation deletes keys one by one under its lock, so the CAS maximum is the only bound. size_t bulkDeleteKeyLimit() const override { return kBulkDeleteMaxKeys; } + size_t conditionalBatchDeleteKeyLimit() const override { return kBulkDeleteMaxKeys; } /// Creates the key when `expected_value` is empty, or replaces the incarnation it names. A /// refused precondition leaves the store unchanged. Value enforcement can be disabled with diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInstrumentedBackend.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInstrumentedBackend.cpp index 4782bd3f10116..90d4b410cee3b 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInstrumentedBackend.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInstrumentedBackend.cpp @@ -154,4 +154,13 @@ void InstrumentedBackend::removeManyWriteOnce(const std::vector & incrementCasEvent(classifyCasNs(key.str()), CasOp::Delete); } +std::vector InstrumentedBackend::removeManyByConditional( + const std::vector & removals, TransportAccess & access) +{ + std::vector results = inner->removeManyByConditional(removals, access); + for (const RawConditionalRemove & removal : removals) + incrementCasEvent(classifyCasNs(removal.key), CasOp::Delete); + return results; +} + } diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInstrumentedBackend.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInstrumentedBackend.h index 5d1f3f36e9536..11fa5d2416533 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInstrumentedBackend.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInstrumentedBackend.h @@ -130,6 +130,8 @@ class InstrumentedBackend final : public Backend /// carried, the request counter says how many requests it took. Out of line, like `publish`, so /// the header need not declare the `CASBulkDeleteRequests` extern. void removeManyWriteOnce(const std::vector & keys, TransportAccess & access) override; + std::vector removeManyByConditional( + const std::vector & removals, TransportAccess & access) override; /// Count a create and a replacement separately, and each of them separately from its refusal: /// they cost the same one request, but a pool whose creates are mostly refused and one whose @@ -162,6 +164,7 @@ class InstrumentedBackend final : public Backend uint64_t attemptTimeoutMs() const override { return inner->attemptTimeoutMs(); } uint64_t attemptEnvelopeMs() const override { return inner->attemptEnvelopeMs(); } size_t bulkDeleteKeyLimit() const override { return inner->bulkDeleteKeyLimit(); } + size_t conditionalBatchDeleteKeyLimit() const override { return inner->conditionalBatchDeleteKeyLimit(); } bool refreshCredentials() override { return inner->refreshCredentials(); } private: diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.cpp index 9e96b1a33efba..8bcab7242eee0 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.cpp @@ -19,9 +19,11 @@ #include +#include #include #include #include +#include #include namespace ProfileEvents @@ -635,6 +637,57 @@ void CasOperation::removeManyWriteOnce(const std::vector & keys, c }); } +std::vector CasOperation::removeManyByConditional( + const std::vector & removals, const Retry & policy) +{ + const size_t limit = std::min(kBulkDeleteMaxKeys, owner.backend->conditionalBatchDeleteKeyLimit()); + if (removals.size() > limit) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CAS removeManyByConditional: {} keys in one chunk, the limit is {}; the consumer chunks its input", + removals.size(), limit); + if (removals.empty()) + return {}; + + std::set keys; + std::vector raw_removals; + raw_removals.reserve(removals.size()); + for (const ConditionalRemove & removal : removals) + { + if (!keys.emplace(removal.key).second) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CAS removeManyByConditional: duplicate key '{}' in one chunk", removal.key); + raw_removals.push_back({removal.key, owner.valueFor(removal.key, removal.expected_token)}); + } + + const Retry::Bound bound = policy.bind(owner.now_ms()); + const String subject = fmt::format("{} (+{} keys)", removals.front().key, removals.size() - 1); + const std::vector raw_results = readLoop( + "removeManyByConditional", subject, policy, bound, [&](auto & access) + { + return owner.backend->removeManyByConditional(raw_removals, access); + }); + if (raw_results.size() != removals.size()) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "CAS removeManyByConditional: backend returned {} results for {} keys", raw_results.size(), removals.size()); + + std::vector results; + results.reserve(raw_results.size()); + for (const Backend::RawRemoval raw : raw_results) + { + switch (raw) + { + case Backend::RawRemoval::Removed: results.push_back(Removal::Removed); break; + case Backend::RawRemoval::Gone: results.push_back(Removal::Gone); break; + case Backend::RawRemoval::Mismatch: results.push_back(Removal::Mismatch); break; + case Backend::RawRemoval::DeleteMarker: + throw Exception(ErrorCodes::CAS_DELETE_MARKER, + "CAS conditional batch removal archived a noncurrent version instead of reclaiming an object: " + "the bucket has object versioning enabled"); + } + } + return results; +} + Removal CasOperation::remove(const String & key, const Etag & seen, const Retry & policy) { return removeUnder(key, owner.valueFor(key, seen), policy, policy.bind(owner.now_ms())); diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.h index 0f1e5696014ca..08f2d3d881fa9 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.h @@ -273,6 +273,12 @@ class CasRequests /// The cap on one `removeManyWriteOnce` chunk -- also the ceiling a batch-delete request can carry. inline constexpr size_t kBulkDeleteMaxKeys = 1000; +struct ConditionalRemove +{ + String key; + Etag expected_token; +}; + /// One admitted operation: the unit a policy, a fence generation and a liveness predicate apply to. /// Move-only, and every request it makes re-checks its admission -- before each attempt, before each /// sleep, and once more after a proven commit, so a write whose fence was lost while it was in flight @@ -328,6 +334,7 @@ class CasOperation /// success. Throws when the policy is exhausted. More keys than the cap is a caller bug: the /// consumer chunks, so that every chunk that succeeded is recorded before a later one can fail. void removeManyWriteOnce(const std::vector & keys, const Retry & policy); + std::vector removeManyByConditional(const std::vector & removals, const Retry & policy); /// The one primitive that reports failure as a value, so the policy reissues on the OUTCOME: /// `Indeterminate` is retried, the four authoritative outcomes return at once, and an /// `Indeterminate` that outlives the bound is returned rather than thrown. A lost fence or a spent diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasThrottlingBackend.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasThrottlingBackend.h index 8f69d4464347d..bb09ca111e1c5 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasThrottlingBackend.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasThrottlingBackend.h @@ -95,6 +95,14 @@ class ThrottlingBackend final : public Backend inner->removeManyWriteOnce(keys, access); } + std::vector removeManyByConditional( + const std::vector & removals, TransportAccess & access) override + { + for (const RawConditionalRemove & removal : removals) + refuseOrPass(removal.key); + return inner->removeManyByConditional(removals, access); + } + std::expected write(const String & key, const String & bytes, const std::optional & expected_value, TransportAccess & access) override { @@ -126,6 +134,7 @@ class ThrottlingBackend final : public Backend uint64_t attemptTimeoutMs() const override { return inner->attemptTimeoutMs(); } uint64_t attemptEnvelopeMs() const override { return inner->attemptEnvelopeMs(); } size_t bulkDeleteKeyLimit() const override { return inner->bulkDeleteKeyLimit(); } + size_t conditionalBatchDeleteKeyLimit() const override { return inner->conditionalBatchDeleteKeyLimit(); } bool refreshCredentials() override { return inner->refreshCredentials(); } void checkPoolPreconditions() override { inner->checkPoolPreconditions(); } void checkSkipAccessCheckSupport() override { inner->checkSkipAccessCheckSupport(); } diff --git a/src/Disks/tests/gtest_cas_bulk_delete_backend.cpp b/src/Disks/tests/gtest_cas_bulk_delete_backend.cpp index 3ce05af764752..39e6f259faca7 100644 --- a/src/Disks/tests/gtest_cas_bulk_delete_backend.cpp +++ b/src/Disks/tests/gtest_cas_bulk_delete_backend.cpp @@ -141,6 +141,26 @@ TEST(CASBulkDeleteBackend, InstrumentedCountsOneRequestAndOneDeletePerKeyClass) EXPECT_EQ(ProfileEvents::global_counters[ProfileEvents::CASRootDelete] - root_before, 1u); } +TEST(CASConditionalBatchDeleteBackend, InstrumentedBackendForwardsMultiKeyConditionalRemove) +{ + auto inner = std::make_shared(); + auto backend = std::make_shared(inner); + CasRequests requests = openRequestsForTest(backend); + CasOperation op = requests.admit(); + const String first_key = "p/conditional/instrumented/first"; + const String second_key = "p/conditional/instrumented/second"; + const Etag first_token = std::get(op.create(first_key, "v1", Retry::once())).etag; + const Etag second_token = std::get(op.create(second_key, "v1", Retry::once())).etag; + + EXPECT_EQ(backend->conditionalBatchDeleteKeyLimit(), kBulkDeleteMaxKeys); + const std::vector results = op.removeManyByConditional( + {{first_key, first_token}, {second_key, second_token}}, Retry::once()); + + EXPECT_EQ(results, (std::vector{Removal::Removed, Removal::Removed})); + EXPECT_FALSE(op.head(first_key, Retry::once()).has_value()); + EXPECT_FALSE(op.head(second_key, Retry::once()).has_value()); +} + #if USE_AWS_S3 TEST(CASBulkDeleteBackend, ThrottlingBackendForwardsStorageLimit) { diff --git a/src/Disks/tests/gtest_cas_bulk_delete_engine.cpp b/src/Disks/tests/gtest_cas_bulk_delete_engine.cpp index 01ace5ad336c3..3afa80c5928c2 100644 --- a/src/Disks/tests/gtest_cas_bulk_delete_engine.cpp +++ b/src/Disks/tests/gtest_cas_bulk_delete_engine.cpp @@ -34,6 +34,12 @@ std::vector manifestKeys(uint32_t count) return keys; } +class UnlimitedConditionalBatchBackend : public InMemoryBackend +{ +public: + size_t conditionalBatchDeleteKeyLimit() const override { return kBulkDeleteMaxKeys + 1; } +}; + } TEST(CASBulkDeleteEngine, AFailedAttemptReissuesTheWholeChunkAndAlreadyDeletedKeysAreSuccess) @@ -70,6 +76,34 @@ TEST(CASBulkDeleteEngine, AnEmptyChunkIsNoRequest) EXPECT_EQ(backend->bulkRemoveCalls(), 0u); } +TEST(CASConditionalBatchDeleteEngine, ReturnsMixedOutcomesInInputOrder) +{ + auto backend = std::make_shared(); + CasRequests requests = openRequestsForTest(backend); + CasOperation op = requests.admit(); + const String removed_key = "p/conditional/removed"; + const String replaced_key = "p/conditional/replaced"; + const String gone_key = "p/conditional/gone"; + + const Etag removed_token = std::get(op.create(removed_key, "v1", Retry::once())).etag; + const Etag stale_token = std::get(op.create(replaced_key, "v1", Retry::once())).etag; + const Etag gone_token = std::get(op.create(gone_key, "v1", Retry::once())).etag; + ASSERT_TRUE(std::holds_alternative(op.replace(replaced_key, "v2", stale_token, Retry::once()))); + ASSERT_EQ(op.remove(gone_key, gone_token, Retry::once()), Removal::Removed); + + const std::vector results = op.removeManyByConditional( + {{removed_key, removed_token}, {replaced_key, stale_token}, {gone_key, gone_token}}, Retry::once()); + + ASSERT_EQ(results.size(), 3u); + EXPECT_EQ(results[0], Removal::Removed); + EXPECT_EQ(results[1], Removal::Mismatch); + EXPECT_EQ(results[2], Removal::Gone); + EXPECT_FALSE(op.head(removed_key, Retry::once()).has_value()); + const auto replaced = op.read(replaced_key, Retry::once()); + ASSERT_TRUE(replaced.has_value()); + EXPECT_EQ(replaced->bytes, "v2"); +} + TEST(CASBulkDeleteEngine, ExactlyOneThousandKeysIsOneRequest) { auto backend = std::make_shared(); @@ -89,3 +123,19 @@ TEST(CASBulkDeleteEngineDeathTest, MoreThanOneThousandKeysIsACallerBug) EXPECT_EQ(backend->bulkRemoveCalls(), 0u); } #endif + +TEST(CASConditionalBatchDeleteEngine, MoreThanOneThousandKeysIsRejectedEvenIfBackendReportsALargerLimit) +{ + auto backend = std::make_shared(); + CasRequests requests = openRequestsForTest(backend); + CasOperation op = requests.admit(); + std::vector removals; + removals.reserve(kBulkDeleteMaxKeys + 1); + for (size_t i = 0; i <= kBulkDeleteMaxKeys; ++i) + { + const String key = "p/conditional/limit/" + std::to_string(i); + removals.push_back({key, std::get(op.create(key, "v", Retry::once())).etag}); + } + + EXPECT_THROW(op.removeManyByConditional(removals, Retry::once()), DB::Exception); +} From 08edf18093ba28bed322e0667a94abced1f725da Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Fri, 9 Oct 2026 18:18:09 +0200 Subject: [PATCH 02/30] Add S3 conditional batch delete seam Signed-off-by: Konstantin Morozov --- .../ObjectStorages/IObjectStorage.cpp | 11 ++++ .../ObjectStorages/IObjectStorage.h | 9 +++ .../ObjectStorages/S3/S3ObjectStorage.cpp | 56 +++++++++++++++++++ .../ObjectStorages/S3/S3ObjectStorage.h | 3 + .../gtest_cas_s3_bulk_delete_fallback.cpp | 32 +++++++++++ 5 files changed, 111 insertions(+) diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.cpp index c763863d1a729..8a902b11c52dc 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.cpp @@ -67,6 +67,17 @@ ObjectStorageIteratorPtr IObjectStorage::iterate( return std::make_shared(std::move(files)); } +std::vector IObjectStorage::removeObjectsIfTokensMatch( + const std::vector & objects, const ObjectStorageControlRequest & request) +{ + if (objects.size() > 1) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, + "Conditional batch object removal is not implemented for {} object storage", getName()); + if (objects.empty()) + return {}; + return {removeObjectIfTokenMatches(objects.front().object, objects.front().expected_token, request)}; +} + ObjectStorageIteratorPtr IObjectStorage::iterate( const std::string & path_prefix, size_t max_keys, diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h b/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h index 2ced78db5b67d..0ed3e296bcbf7 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h @@ -297,6 +297,12 @@ struct ConditionalRemoveResult bool created_delete_marker = false; /// backend reported a versioning delete marker }; +struct ConditionalRemoveObject +{ + StoredObject object; + std::string expected_token; +}; + /// Base class for all object storages which implement some subset of ordinary filesystem operations. /// /// Examples of object storages are S3, Azure Blob Storage, HDFS. @@ -447,6 +453,9 @@ class IObjectStorage virtual ConditionalRemoveResult removeObjectIfTokenMatches( const StoredObject & object, const std::string & etag, const ObjectStorageControlRequest & request); + virtual std::vector removeObjectsIfTokensMatch( + const std::vector & objects, const ObjectStorageControlRequest & request); + /// Removes every object in ONE request with no per-key precondition; an absent object is success. /// Content-addressed callers use it for write-once keys only, at most 1000 per call. Throws on a /// request-level failure and on any per-key error other than "not found", naming the failed keys. diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp index 9c2eb04456f35..b40f6cee3a74b 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp @@ -638,6 +638,62 @@ ConditionalRemoveResult S3ObjectStorage::removeObjectIfTokenMatches( object, etag, clientForRetryProfile(request), request.attempt_number); }); } +std::vector S3ObjectStorage::removeObjectsIfTokensMatch( + const std::vector & objects, const ObjectStorageControlRequest & request) +{ + if (objects.empty()) + return {}; + if (objects.size() == 1) + return {removeObjectIfTokenMatches(objects.front().object, objects.front().expected_token, request)}; + + std::vector identifiers; + identifiers.reserve(objects.size()); + for (const ConditionalRemoveObject & object : objects) + { + Aws::S3::Model::ObjectIdentifier identifier; + identifier.SetKey(object.object.remote_path); + identifier.SetETag(object.expected_token); + identifiers.push_back(std::move(identifier)); + } + Aws::S3::Model::Delete delete_request; + delete_request.SetObjects(std::move(identifiers)); + delete_request.SetQuiet(false); + S3::DeleteObjectsRequest request_s3; + request_s3.SetBucket(uri.bucket); + request_s3.SetDelete(std::move(delete_request)); + if (request.attempt_number != 0) + S3::setClickhouseAttemptNumber(request_s3, request.attempt_number); + ProfileEvents::increment(ProfileEvents::DiskS3DeleteObjects); + const auto outcome = clientForRetryProfile(request)->DeleteObjects(request_s3); + if (!outcome.IsSuccess()) + throw S3Exception(outcome.GetError().GetErrorType(), "{} while conditionally removing objects from S3", outcome.GetError().GetMessage()); + + std::vector results; + results.reserve(objects.size()); + for (const ConditionalRemoveObject & object : objects) + { + const auto deleted = std::find_if(outcome.GetResult().GetDeleted().begin(), outcome.GetResult().GetDeleted().end(), + [&](const auto & entry) { return entry.GetKey() == object.object.remote_path; }); + if (deleted != outcome.GetResult().GetDeleted().end()) + { + results.push_back({ConditionalRemoveOutcome::Removed, deleted->GetDeleteMarker()}); + continue; + } + const auto error = std::find_if(outcome.GetResult().GetErrors().begin(), outcome.GetResult().GetErrors().end(), + [&](const auto & entry) { return entry.GetKey() == object.object.remote_path; }); + if (error == outcome.GetResult().GetErrors().end()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "S3 conditional DeleteObjects response omitted key {}", object.object.remote_path); + const auto code = classifyDeleteObjectsErrorCode(error->GetCode()); + if (error->GetCode() == "PreconditionFailed") + results.push_back({ConditionalRemoveOutcome::TokenMismatch, false}); + else if (S3::isNotFoundError(code)) + results.push_back({ConditionalRemoveOutcome::NotFound, false}); + else + throw S3Exception(code, "{} while conditionally removing object {} from S3", error->GetMessage(), error->GetKey()); + } + return results; +} + ConditionalRemoveResult S3ObjectStorage::removeObjectIfTokenMatchesImpl( const StoredObject & object, const std::string & etag, const std::shared_ptr & used_client, size_t attempt_seed) diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h index 50229693f275f..b82635ba57783 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h @@ -130,6 +130,9 @@ class S3ObjectStorage : public IObjectStorage ConditionalRemoveResult removeObjectIfTokenMatches( const StoredObject & object, const std::string & etag, const ObjectStorageControlRequest & request) override; + std::vector removeObjectsIfTokensMatch( + const std::vector & objects, const ObjectStorageControlRequest & request) override; + /// One `DeleteObjects` for the given objects (the caller chunks to at most 1000); absence is success. /// Exactly one object is always a plain `DeleteObject` instead (never gated on `s3_capabilities`: a /// single physical request per call, so there is nothing here for that capability to say no to). diff --git a/src/Disks/tests/gtest_cas_s3_bulk_delete_fallback.cpp b/src/Disks/tests/gtest_cas_s3_bulk_delete_fallback.cpp index d5bbf95a8dbab..61e646ac98427 100644 --- a/src/Disks/tests/gtest_cas_s3_bulk_delete_fallback.cpp +++ b/src/Disks/tests/gtest_cas_s3_bulk_delete_fallback.cpp @@ -199,6 +199,17 @@ void sendDeleteObjectsResult(Poco::Net::HTTPServerResponse & response, const std sendXml(response, Poco::Net::HTTPResponse::HTTP_OK, body); } +void sendConditionalDeleteObjectsResult(Poco::Net::HTTPServerResponse & response) +{ + const std::string body = + "" + "" + "first" + "secondPreconditionFailedstale" + ""; + sendXml(response, Poco::Net::HTTPResponse::HTTP_OK, body); +} + std::shared_ptr makeStorageForTest( const std::string & endpoint, const DB::S3Capabilities & capabilities, std::optional objects_chunk_size_to_delete = {}) { @@ -294,6 +305,27 @@ TEST(S3BulkDeleteFallback, PerKeyErrorsWithinASuccessfulBatchAreUnchanged) EXPECT_EQ(server.countMethod("DELETE"), 0u); } +TEST(CASS3ConditionalBatchDelete, ReturnsPerKeyOutcomesInInputOrder) +{ + (void)contextForTest(); + + ScriptedS3Server server([](const Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) + { + ASSERT_EQ(request.getMethod(), "POST"); + sendConditionalDeleteObjectsResult(response); + }); + auto storage = makeStorageForTest(server.getUrl(), DB::S3Capabilities{}); + + const auto results = storage->removeObjectsIfTokensMatch( + {{DB::StoredObject("first"), "etag-first"}, {DB::StoredObject("second"), "etag-second"}}, + DB::ObjectStorageControlRequest{}); + + ASSERT_EQ(results.size(), 2u); + EXPECT_EQ(results[0].outcome, DB::ConditionalRemoveOutcome::Removed); + EXPECT_EQ(results[1].outcome, DB::ConditionalRemoveOutcome::TokenMismatch); + EXPECT_EQ(server.countMethod("POST"), 1u); +} + TEST(S3BulkDeleteFallback, UnsupportedBatchReplyRecordsCapabilityFalseAndThrowsNotImplemented) { (void)contextForTest(); From 952ae3c9e043aac72d2ba7eba0b183c57074c487 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Fri, 9 Oct 2026 19:00:05 +0200 Subject: [PATCH 03/30] Batch conditional CAS blob redelete Signed-off-by: Konstantin Morozov --- .../ContentAddressed/Backend/CasBackend.h | 1 + .../Backend/CasInMemoryBackend.h | 4 +- .../Backend/CasInstrumentedBackend.h | 1 + .../Backend/CasObjectStorageBackend.cpp | 54 +++++++++++++++++++ .../Backend/CasObjectStorageBackend.h | 5 ++ .../Backend/CasThrottlingBackend.h | 1 + .../ContentAddressed/Gc/CasGc.cpp | 49 ++++++++++++++++- .../ContentAddressed/Gc/CasGc.h | 1 + .../tests/gtest_cas_bulk_delete_backend.cpp | 1 + .../tests/gtest_cas_bulk_delete_engine.cpp | 2 + .../gtest_cas_gc_redelete_concurrency.cpp | 36 ++++++++++++- 11 files changed, 151 insertions(+), 4 deletions(-) diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasBackend.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasBackend.h index d6d24c23c4f1a..2735bb55eb14a 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasBackend.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasBackend.h @@ -226,6 +226,7 @@ class Backend virtual size_t bulkDeleteKeyLimit() const { return 1; } virtual size_t conditionalBatchDeleteKeyLimit() const { return 1; } + virtual void pinConditionalBatchDeleteSupport(bool) {} /// Authoritative, cache-bypassing probe of one key -- see `ProbeOutcome`. DEFAULT (used by every /// backend without sharper raw-error evidence, e.g. `InMemoryBackend`): derived from `head`/`read` diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInMemoryBackend.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInMemoryBackend.h index 51f0ae66bedb9..b7a279724390b 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInMemoryBackend.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInMemoryBackend.h @@ -64,7 +64,8 @@ class InMemoryBackend : public Backend /// The emulation deletes keys one by one under its lock, so the CAS maximum is the only bound. size_t bulkDeleteKeyLimit() const override { return kBulkDeleteMaxKeys; } - size_t conditionalBatchDeleteKeyLimit() const override { return kBulkDeleteMaxKeys; } + size_t conditionalBatchDeleteKeyLimit() const override { return conditional_batch_delete_supported ? kBulkDeleteMaxKeys : 1; } + void pinConditionalBatchDeleteSupport(bool supported) override { conditional_batch_delete_supported = supported; } /// Creates the key when `expected_value` is empty, or replaces the incarnation it names. A /// refused precondition leaves the store unchanged. Value enforcement can be disabled with @@ -230,6 +231,7 @@ class InMemoryBackend : public Backend std::vector armed_bulk_remove_failures_; std::function before_bulk_remove_hook_; size_t bulk_remove_calls_ = 0; + std::atomic_bool conditional_batch_delete_supported = false; }; } diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInstrumentedBackend.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInstrumentedBackend.h index 11fa5d2416533..041ad1d7b5198 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInstrumentedBackend.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInstrumentedBackend.h @@ -165,6 +165,7 @@ class InstrumentedBackend final : public Backend uint64_t attemptEnvelopeMs() const override { return inner->attemptEnvelopeMs(); } size_t bulkDeleteKeyLimit() const override { return inner->bulkDeleteKeyLimit(); } size_t conditionalBatchDeleteKeyLimit() const override { return inner->conditionalBatchDeleteKeyLimit(); } + void pinConditionalBatchDeleteSupport(bool supported) override { inner->pinConditionalBatchDeleteSupport(supported); } bool refreshCredentials() override { return inner->refreshCredentials(); } private: diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasObjectStorageBackend.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasObjectStorageBackend.cpp index 9d8dead79945f..bbbb3cdabc06a 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasObjectStorageBackend.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasObjectStorageBackend.cpp @@ -1027,6 +1027,18 @@ size_t ObjectStorageBackend::bulkDeleteKeyLimit() const return mode == Mode::Native ? object_storage->batchDeleteKeyLimit() : kBulkDeleteMaxKeys; } +size_t ObjectStorageBackend::conditionalBatchDeleteKeyLimit() const +{ + if (mode != Mode::Native || !conditional_batch_delete_supported.load()) + return 1; + return object_storage->batchDeleteKeyLimit(); +} + +void ObjectStorageBackend::pinConditionalBatchDeleteSupport(bool supported) +{ + conditional_batch_delete_supported.store(supported); +} + void ObjectStorageBackend::removeManyWriteOnce(const std::vector & keys, TransportAccess & access) { if (keys.empty()) @@ -1054,6 +1066,48 @@ void ObjectStorageBackend::removeManyWriteOnce(const std::vector & } } +std::vector ObjectStorageBackend::removeManyByConditional( + const std::vector & removals, TransportAccess & access) +{ + if (removals.empty()) + return {}; + if (mode != Mode::Native) + { + std::vector results; + results.reserve(removals.size()); + for (const RawConditionalRemove & removal : removals) + results.push_back(removeUnder(removal.key, removal.expected_token, controlRequest(access.attemptNo()))); + return results; + } + + std::vector objects; + objects.reserve(removals.size()); + for (const RawConditionalRemove & removal : removals) + { + if (!isValidTokenValue(dialect(), removal.expected_token)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "CAS backend: malformed conditional removal token for {}", removal.key); + objects.push_back({StoredObject(removal.key), removal.expected_token}); + } + const auto storage_results = object_storage->removeObjectsIfTokensMatch(objects, controlRequest(access.attemptNo())); + if (storage_results.size() != removals.size()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "CAS backend: conditional batch removal returned {} results for {} keys", storage_results.size(), removals.size()); + + std::vector results; + results.reserve(storage_results.size()); + for (const ConditionalRemoveResult & result : storage_results) + { + switch (result.outcome) + { + case ConditionalRemoveOutcome::Removed: + results.push_back(result.created_delete_marker ? RawRemoval::DeleteMarker : RawRemoval::Removed); + break; + case ConditionalRemoveOutcome::TokenMismatch: results.push_back(RawRemoval::Mismatch); break; + case ConditionalRemoveOutcome::NotFound: results.push_back(RawRemoval::Gone); break; + } + } + return results; +} + Backend::RawListPage ObjectStorageBackend::list(const String & prefix, const String & cursor, size_t limit, TransportAccess & access) { return listUnder(prefix, cursor, limit, controlRequest(access.attemptNo())); diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasObjectStorageBackend.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasObjectStorageBackend.h index f9fa31e55fbb9..b1caef8ef10c9 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasObjectStorageBackend.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasObjectStorageBackend.h @@ -101,10 +101,14 @@ class ObjectStorageBackend final : public Backend /// under the control-plane profile; `EmulatedSingleProcess` deletes each present key under the /// emulation lock with the same token bookkeeping as the single-key delete. void removeManyWriteOnce(const std::vector & keys, TransportAccess & access) override; + std::vector removeManyByConditional( + const std::vector & removals, TransportAccess & access) override; /// Native: the object storage's own limit. Emulated: `kBulkDeleteMaxKeys`, since this backend deletes /// the keys one by one under its emulation lock. size_t bulkDeleteKeyLimit() const override; + size_t conditionalBatchDeleteKeyLimit() const override; + void pinConditionalBatchDeleteSupport(bool supported) override; /// Native mints its store's own dialect (ETag or GCS generation); the emulated adapter mints its /// own values. Dialect dialect() const override { return mode == Mode::Native ? native_token_type : Dialect::Emulated; } @@ -216,6 +220,7 @@ class ObjectStorageBackend final : public Backend private: const ObjectStoragePtr object_storage; const Mode mode; + std::atomic_bool conditional_batch_delete_supported = false; Dialect native_token_type = Dialect::ETag; /// See the constructor: what the READ-class requests (read, head, list, remove) carry. const bool single_attempt_control_plane; diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasThrottlingBackend.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasThrottlingBackend.h index bb09ca111e1c5..9789cb55c9f72 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasThrottlingBackend.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasThrottlingBackend.h @@ -135,6 +135,7 @@ class ThrottlingBackend final : public Backend uint64_t attemptEnvelopeMs() const override { return inner->attemptEnvelopeMs(); } size_t bulkDeleteKeyLimit() const override { return inner->bulkDeleteKeyLimit(); } size_t conditionalBatchDeleteKeyLimit() const override { return inner->conditionalBatchDeleteKeyLimit(); } + void pinConditionalBatchDeleteSupport(bool supported) override { inner->pinConditionalBatchDeleteSupport(supported); } bool refreshCredentials() override { return inner->refreshCredentials(); } void checkPoolPreconditions() override { inner->checkPoolPreconditions(); } void checkSkipAccessCheckSupport() override { inner->checkSkipAccessCheckSupport(); } diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp index d95e8a3f78405..7f54fb7a41669 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp @@ -584,7 +584,12 @@ Gc::RedeleteIo Gc::performRedeleteIo(const RetiredEntry & entry, const Layout & /// already gone. const std::optional observed = op.head(io.blob_key, Retry::standard()); if (observed) - io.del = entry.token.matches(observed->etag) ? op.remove(io.blob_key, observed->etag, Retry::standard()) : Removal::Mismatch; + { + if (entry.token.matches(observed->etag)) + io.expected_token = observed->etag; + else + io.del = Removal::Mismatch; + } return io; } @@ -728,6 +733,48 @@ void Gc::redeleteBlobs( } } + std::vector matched; + for (size_t i = 0; i < scheduled; ++i) + if (!io_errors[i] && io_results[i].expected_token) + matched.push_back(i); + const size_t chunk_keys = std::max(1, std::min(kBulkDeleteMaxKeys, store->poolBackendPtr()->conditionalBatchDeleteKeyLimit())); + ThreadPoolCallbackRunnerLocal delete_runner(pool, ThreadName::CAS_GC_REDELETE); + std::vector::Task>> delete_handles; + for (size_t begin = 0; begin < matched.size(); begin += chunk_keys) + { + const size_t end = std::min(matched.size(), begin + chunk_keys); + std::vector removals; + removals.reserve(end - begin); + for (size_t pos = begin; pos < end; ++pos) + { + const size_t index = matched[pos]; + removals.push_back({io_results[index].blob_key, *io_results[index].expected_token}); + } + delete_handles.emplace_back(delete_runner.enqueueAndGiveOwnership( + [begin, end, chunk_removals = std::move(removals), &matched, &io_results, pool_store = store, resumed_generation = op.generation()] + { + CasOperation job_op = pool_store->openRequests().resume(resumed_generation); + const auto results = job_op.removeManyByConditional(chunk_removals, Retry::standard()); + for (size_t pos = begin; pos < end; ++pos) + io_results[matched[pos]].del = results[pos - begin]; + })); + } + ThreadPoolCallbackRunnerLocal::waitForAllToFinish(delete_handles); + for (size_t chunk = 0; chunk < delete_handles.size(); ++chunk) + { + try + { + delete_handles[chunk]->future.get(); + } + catch (...) + { + const size_t begin = chunk * chunk_keys; + const size_t end = std::min(matched.size(), begin + chunk_keys); + for (size_t pos = begin; pos < end; ++pos) + io_errors[matched[pos]] = std::current_exception(); + } + } + std::optional first_error; for (size_t i = 0; i < scheduled; ++i) { diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h index 3ffb51d180724..7dbb41a136204 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h @@ -744,6 +744,7 @@ class Gc struct RedeleteIo { String blob_key; + std::optional expected_token; Removal del = Removal::Gone; }; diff --git a/src/Disks/tests/gtest_cas_bulk_delete_backend.cpp b/src/Disks/tests/gtest_cas_bulk_delete_backend.cpp index 39e6f259faca7..96cbc00d979d8 100644 --- a/src/Disks/tests/gtest_cas_bulk_delete_backend.cpp +++ b/src/Disks/tests/gtest_cas_bulk_delete_backend.cpp @@ -144,6 +144,7 @@ TEST(CASBulkDeleteBackend, InstrumentedCountsOneRequestAndOneDeletePerKeyClass) TEST(CASConditionalBatchDeleteBackend, InstrumentedBackendForwardsMultiKeyConditionalRemove) { auto inner = std::make_shared(); + inner->pinConditionalBatchDeleteSupport(true); auto backend = std::make_shared(inner); CasRequests requests = openRequestsForTest(backend); CasOperation op = requests.admit(); diff --git a/src/Disks/tests/gtest_cas_bulk_delete_engine.cpp b/src/Disks/tests/gtest_cas_bulk_delete_engine.cpp index 3afa80c5928c2..48a16b7a8df87 100644 --- a/src/Disks/tests/gtest_cas_bulk_delete_engine.cpp +++ b/src/Disks/tests/gtest_cas_bulk_delete_engine.cpp @@ -79,6 +79,7 @@ TEST(CASBulkDeleteEngine, AnEmptyChunkIsNoRequest) TEST(CASConditionalBatchDeleteEngine, ReturnsMixedOutcomesInInputOrder) { auto backend = std::make_shared(); + backend->pinConditionalBatchDeleteSupport(true); CasRequests requests = openRequestsForTest(backend); CasOperation op = requests.admit(); const String removed_key = "p/conditional/removed"; @@ -127,6 +128,7 @@ TEST(CASBulkDeleteEngineDeathTest, MoreThanOneThousandKeysIsACallerBug) TEST(CASConditionalBatchDeleteEngine, MoreThanOneThousandKeysIsRejectedEvenIfBackendReportsALargerLimit) { auto backend = std::make_shared(); + backend->pinConditionalBatchDeleteSupport(true); CasRequests requests = openRequestsForTest(backend); CasOperation op = requests.admit(); std::vector removals; diff --git a/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp b/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp index 5fbdd3eb8e304..6145f2b27c775 100644 --- a/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp +++ b/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp @@ -82,6 +82,15 @@ class WorkerFaultBackend : public InMemoryBackend return InMemoryBackend::remove(key, expected_value, access); } + std::vector removeManyByConditional( + const std::vector & removals, TransportAccess & access) override + { + for (const RawConditionalRemove & removal : removals) + if (takeFault(remove_keys, removal.key)) + throw std::runtime_error("injected worker remove fault for key " + removal.key); + return InMemoryBackend::removeManyByConditional(removals, access); + } + std::optional head(const String & key, TransportAccess & access) override { if (takeFault(head_keys, key)) @@ -165,6 +174,16 @@ class BlobRemoveWitnessBackend : public InMemoryBackend return result; } + std::vector removeManyByConditional( + const std::vector & removals, TransportAccess & access) override + { + std::vector results; + results.reserve(removals.size()); + for (const RawConditionalRemove & removal : removals) + results.push_back(remove(removal.key, removal.expected_token, access)); + return results; + } + private: std::set blob_keys; size_t k_overlap = 1; @@ -520,6 +539,21 @@ class RemoveRaceBackend : public InMemoryBackend bool replacementCommitted() const { return replacement_committed.load(); } RawRemoval remove(const String & key, const String & expected_value, TransportAccess & access) override + { + injectReplacement(key, access); + return InMemoryBackend::remove(key, expected_value, access); + } + + std::vector removeManyByConditional( + const std::vector & removals, TransportAccess & access) override + { + for (const RawConditionalRemove & removal : removals) + injectReplacement(removal.key, access); + return InMemoryBackend::removeManyByConditional(removals, access); + } + +private: + void injectReplacement(const String & key, TransportAccess & access) { if (armed.load() && key == raced_key && armed.exchange(false)) { @@ -528,10 +562,8 @@ class RemoveRaceBackend : public InMemoryBackend if (current) replacement_committed.store(InMemoryBackend::write(key, current->bytes, current->value, access).has_value()); } - return InMemoryBackend::remove(key, expected_value, access); } -private: String raced_key; std::thread::id owner; std::atomic armed{false}; From 30942fded28c09948f5de8b7342e9723ec7b1a79 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Fri, 9 Oct 2026 19:03:11 +0200 Subject: [PATCH 04/30] Validate conditional S3 batch responses Signed-off-by: Konstantin Morozov --- .../ObjectStorages/S3/S3ObjectStorage.cpp | 24 +++++++++++++++++++ .../gtest_cas_s3_bulk_delete_fallback.cpp | 24 +++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp index b40f6cee3a74b..d7b61eda808c7 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp @@ -43,6 +43,8 @@ #include #include +#include + namespace ProfileEvents { extern const Event S3ListObjects; @@ -668,6 +670,28 @@ std::vector S3ObjectStorage::removeObjectsIfTokensMatch if (!outcome.IsSuccess()) throw S3Exception(outcome.GetError().GetErrorType(), "{} while conditionally removing objects from S3", outcome.GetError().GetMessage()); + std::unordered_set expected_keys; + expected_keys.reserve(objects.size()); + for (const ConditionalRemoveObject & object : objects) + expected_keys.insert(object.object.remote_path); + + std::unordered_set response_keys; + response_keys.reserve(objects.size()); + for (const auto & deleted : outcome.GetResult().GetDeleted()) + { + const String key = deleted.GetKey(); + if (!expected_keys.contains(key) || !response_keys.insert(key).second) + throw Exception(ErrorCodes::LOGICAL_ERROR, "S3 conditional DeleteObjects response has unexpected or duplicate key {}", key); + } + for (const auto & error : outcome.GetResult().GetErrors()) + { + const String key = error.GetKey(); + if (!expected_keys.contains(key) || !response_keys.insert(key).second) + throw Exception(ErrorCodes::LOGICAL_ERROR, "S3 conditional DeleteObjects response has unexpected or duplicate key {}", key); + } + if (response_keys.size() != objects.size()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "S3 conditional DeleteObjects response omitted keys"); + std::vector results; results.reserve(objects.size()); for (const ConditionalRemoveObject & object : objects) diff --git a/src/Disks/tests/gtest_cas_s3_bulk_delete_fallback.cpp b/src/Disks/tests/gtest_cas_s3_bulk_delete_fallback.cpp index 61e646ac98427..1a779be23f55c 100644 --- a/src/Disks/tests/gtest_cas_s3_bulk_delete_fallback.cpp +++ b/src/Disks/tests/gtest_cas_s3_bulk_delete_fallback.cpp @@ -326,6 +326,30 @@ TEST(CASS3ConditionalBatchDelete, ReturnsPerKeyOutcomesInInputOrder) EXPECT_EQ(server.countMethod("POST"), 1u); } +TEST(CASS3ConditionalBatchDelete, RejectsDuplicateResponseKey) +{ + (void)contextForTest(); + + ScriptedS3Server server([](const Poco::Net::HTTPServerRequest &, Poco::Net::HTTPServerResponse & response) + { + const std::string body = + "" + "" + "first" + "first" + "secondPreconditionFailedstale" + ""; + sendXml(response, Poco::Net::HTTPResponse::HTTP_OK, body); + }); + auto storage = makeStorageForTest(server.getUrl(), DB::S3Capabilities{}); + + EXPECT_THROW( + storage->removeObjectsIfTokensMatch( + {{DB::StoredObject("first"), "etag-first"}, {DB::StoredObject("second"), "etag-second"}}, + DB::ObjectStorageControlRequest{}), + DB::Exception); +} + TEST(S3BulkDeleteFallback, UnsupportedBatchReplyRecordsCapabilityFalseAndThrowsNotImplemented) { (void)contextForTest(); From f75841b59ddc17e93399a438be3a2758baa3dd9e Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Fri, 9 Oct 2026 19:07:41 +0200 Subject: [PATCH 05/30] Record conditional GC delete batches Signed-off-by: Konstantin Morozov --- .../ContentAddressed/Gc/CasGc.cpp | 11 +++++++-- .../ContentAddressed/Gc/CasGc.h | 1 + .../gtest_cas_gc_redelete_concurrency.cpp | 23 ++++++++++++++++++- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp index 7f54fb7a41669..df20d62954540 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp @@ -737,7 +737,10 @@ void Gc::redeleteBlobs( for (size_t i = 0; i < scheduled; ++i) if (!io_errors[i] && io_results[i].expected_token) matched.push_back(i); - const size_t chunk_keys = std::max(1, std::min(kBulkDeleteMaxKeys, store->poolBackendPtr()->conditionalBatchDeleteKeyLimit())); + const size_t chunk_keys = std::clamp( + std::min(store->poolConfig().gc_bulk_delete_chunk_keys, store->poolBackendPtr()->conditionalBatchDeleteKeyLimit()), + 1, + kBulkDeleteMaxKeys); ThreadPoolCallbackRunnerLocal delete_runner(pool, ThreadName::CAS_GC_REDELETE); std::vector::Task>> delete_handles; for (size_t begin = 0; begin < matched.size(); begin += chunk_keys) @@ -758,6 +761,7 @@ void Gc::redeleteBlobs( for (size_t pos = begin; pos < end; ++pos) io_results[matched[pos]].del = results[pos - begin]; })); + ++ctx.delete_batches; } ThreadPoolCallbackRunnerLocal::waitForAllToFinish(delete_handles); for (size_t chunk = 0; chunk < delete_handles.size(); ++chunk) @@ -1100,10 +1104,12 @@ RoundReport Gc::runRegularRound(std::function on_lease_acquired, bool al const uint64_t graduated_before = report.graduated; const uint64_t redelete_failed_before = report.redelete_failed; uint64_t redelete_jobs_scheduled = 0; + uint64_t redelete_delete_batches = 0; const auto record_redelete_job_metrics = [&] { pending_deletes_timer->metric("jobs_scheduled", redelete_jobs_scheduled); pending_deletes_timer->metric("jobs_failed", report.redelete_failed - redelete_failed_before); + pending_deletes_timer->metric("delete_batches", redelete_delete_batches); }; SCOPE_EXIT_SAFE({ if (pending_deletes_timer) @@ -1137,7 +1143,8 @@ RoundReport Gc::runRegularRound(std::function on_lease_acquired, bool al .round_work_budget = round_work_budget, .report = report, .outcomes = outcomes, - .jobs_scheduled = redelete_jobs_scheduled}; + .jobs_scheduled = redelete_jobs_scheduled, + .delete_batches = redelete_delete_batches}; redeleteBlobs(redelete_ctx, redelete_now, *io_pool, store->poolConfig().gc_io_concurrency, layout, op); for (const RetiredEntry & entry : merge.spared) { diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h index 7dbb41a136204..080d7151bbc96 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h @@ -763,6 +763,7 @@ class Gc RoundReport & report; std::map & outcomes; uint64_t & jobs_scheduled; + uint64_t & delete_batches; }; static RedeleteIo performRedeleteIo(const RetiredEntry & entry, const Layout & layout, CasOperation & op); diff --git a/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp b/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp index 6145f2b27c775..e277ebca41822 100644 --- a/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp +++ b/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp @@ -203,7 +203,8 @@ PoolPtr openPoolWithIoConcurrency( uint64_t gc_shards = 1, uint64_t outcome_entry_budget = 0, CasEventSink event_sink = {}, - std::optional refuse_at_for_test = std::nullopt) + std::optional refuse_at_for_test = std::nullopt, + uint64_t gc_bulk_delete_chunk_keys = kBulkDeleteMaxKeys) { PoolConfig config{.pool_prefix = "p", .server_root_id = "test"}; config.gc_io_concurrency = concurrency; @@ -211,6 +212,7 @@ PoolPtr openPoolWithIoConcurrency( config.gc_round_outcome_entry_budget = outcome_entry_budget; config.event_sink = std::move(event_sink); config.gc_io_pool_refuse_at_for_test = refuse_at_for_test; + config.gc_bulk_delete_chunk_keys = gc_bulk_delete_chunk_keys; return Pool::open(std::move(backend), std::move(config)); } @@ -381,6 +383,25 @@ TEST(CASGCRedeleteConcurrency, ParallelRedeleteReclaimsEveryBlob) EXPECT_EQ(rows->total("jobs_failed"), 0u); } +TEST(CASGCRedeleteConcurrency, ConditionalDeleteBatchesAreRecorded) +{ + auto backend = std::make_shared(); + backend->pinConditionalBatchDeleteSupport(true); + auto store = openPoolWithIoConcurrency(backend, 4, 1, 0, {}, std::nullopt, 2); + Gc gc(store, kGc); + publishThenDrop(*backend, store, gc); + ASSERT_NO_FATAL_FAILURE(driveUntilAllGraduated(*backend, gc, store, kReplacedBlob)); + + auto rows = std::make_shared(); + gc.setPhaseSink(recordPendingDeletes(rows)); + const RoundReport rep = runRegularRoundReclaiming(gc); + store->renewWatermarkOnce(); + + EXPECT_EQ(rep.deleted, kBlobs); + ASSERT_EQ(rows->rows.size(), 1u); + EXPECT_EQ(rows->total("delete_batches"), 3u); +} + TEST(CASGCRedeleteConcurrency, BlobDeletesActuallyOverlap) { auto backend = std::make_shared(); From f8f63499cd3a79a7c0e669544354424562a943b3 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Fri, 9 Oct 2026 19:09:53 +0200 Subject: [PATCH 06/30] Retry conditional S3 batch after credential refresh Signed-off-by: Konstantin Morozov --- .../ObjectStorages/S3/S3ObjectStorage.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp index d7b61eda808c7..c3ccda5912b03 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp @@ -665,8 +665,11 @@ std::vector S3ObjectStorage::removeObjectsIfTokensMatch request_s3.SetDelete(std::move(delete_request)); if (request.attempt_number != 0) S3::setClickhouseAttemptNumber(request_s3, request.attempt_number); - ProfileEvents::increment(ProfileEvents::DiskS3DeleteObjects); - const auto outcome = clientForRetryProfile(request)->DeleteObjects(request_s3); + const auto outcome = refreshAndRetryOnExpiredCredentials([&] + { + ProfileEvents::increment(ProfileEvents::DiskS3DeleteObjects); + return clientForRetryProfile(request)->DeleteObjects(request_s3); + }); if (!outcome.IsSuccess()) throw S3Exception(outcome.GetError().GetErrorType(), "{} while conditionally removing objects from S3", outcome.GetError().GetMessage()); From 2b2b04c8f0f619bb7ecc4aee9a36ebba0d304606 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Mon, 12 Oct 2026 11:33:53 +0200 Subject: [PATCH 07/30] Probe conditional batch delete capability Signed-off-by: Konstantin Morozov --- .../ContentAddressed/Backend/CasProbe.cpp | 34 ++++++++++-- .../ContentAddressed/Backend/CasProbe.h | 7 ++- .../ContentAddressed/Backend/CasRequests.cpp | 12 +++++ .../ContentAddressed/Backend/CasRequests.h | 11 ++++ .../ContentAddressed/Pool/CasPool.cpp | 4 +- .../gtest_cas_gc_redelete_concurrency.cpp | 1 + src/Disks/tests/gtest_cas_probe.cpp | 53 ++++++++++++++++++- 7 files changed, 114 insertions(+), 8 deletions(-) diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasProbe.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasProbe.cpp index 481910f1d91f7..20d08f9fbdef6 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasProbe.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasProbe.cpp @@ -44,11 +44,13 @@ Removal removeOrReportDeleteMarker(CasOperation & op, const String & key, const } -void runCapabilityProbe(CasOperation & op, const String & probe_prefix) +CapabilityProbeResult runCapabilityProbe(CasOperation & op, const String & probe_prefix) { // Sub-directory style ("probe_prefix/token") ensures that list(probe_prefix, …) works for both the // in-memory backend (prefix match) and the LocalObjectStorage backend (directory listing). const String key = probe_prefix + "/token"; + const String first_key = probe_prefix + "/batch-first"; + const String second_key = probe_prefix + "/batch-second"; // Best-effort cleanup — runs at function exit regardless of outcome. auto cleanup = [&]() noexcept @@ -59,13 +61,17 @@ void runCapabilityProbe(CasOperation & op, const String & probe_prefix) // constructible here — the gate below is the only way to reach `remove` at all. try { - const auto h = op.head(key, Retry::standard()); - if (h) - op.remove(key, h->etag, Retry::standard()); + for (const String * cleanup_key : {&key, &first_key, &second_key}) + { + const auto h = op.head(*cleanup_key, Retry::standard()); + if (h) + op.remove(*cleanup_key, h->etag, Retry::standard()); + } } catch (...) {} /// NOLINT(bugprone-empty-catch) }; + CapabilityProbeResult result; try { // ---- Step 1: create fresh -> Committed; read-after-write returns the bytes. ---- @@ -195,6 +201,25 @@ void runCapabilityProbe(CasOperation & op, const String & probe_prefix) "CasProbe: list-after-delete failed — the deleted probe key '{}' is still visible in the listing under prefix '{}'", key, probe_prefix); } + + const Etag first_token = std::get(op.create(first_key, "probe-batch-first", Retry::standard())).etag; + const Etag second_token = std::get(op.create(second_key, "probe-batch-second-v1", Retry::standard())).etag; + const Etag second_replacement = std::get(op.replace(second_key, "probe-batch-second-v2", second_token, Retry::standard())).etag; + try + { + const std::vector removals = op.removeManyByConditionalForProbe( + {{first_key, first_token}, {second_key, second_token}}, Retry::standard()); + if (removals.size() == 2 && removals[0] == Removal::Removed && removals[1] == Removal::Mismatch) + { + const auto second = op.read(second_key, Retry::standard()); + result.conditional_batch_delete_supported = second && second->bytes == "probe-batch-second-v2"; + } + } + catch (const DB::Exception & e) + { + if (e.code() != DB::ErrorCodes::NOT_IMPLEMENTED) + throw; + } } catch (...) { @@ -206,6 +231,7 @@ void runCapabilityProbe(CasOperation & op, const String & probe_prefix) // Normal-exit cleanup (the key was already deleted in step 7). Call cleanup anyway to handle any // partial state edge cases — it is a no-op if the key is gone. cleanup(); + return result; } } diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasProbe.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasProbe.h index 99ec36581af29..783261ddbcd27 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasProbe.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasProbe.h @@ -5,6 +5,11 @@ namespace DB::Cas { +struct CapabilityProbeResult +{ + bool conditional_batch_delete_supported = false; +}; + /// Run the capability battery against `op`, using throwaway keys under `probe_prefix`. /// /// `op` must already be admitted. The two store-level precondition hooks — @@ -31,6 +36,6 @@ namespace DB::Cas /// Cleanup of the probe key is best-effort and runs unconditionally: after the battery completes, or on /// the failure path immediately before the check-failing exception is rethrown. Cleanup itself suppresses /// exceptions so that it cannot hide the capability-check failure. -void runCapabilityProbe(CasOperation & op, const String & probe_prefix); +CapabilityProbeResult runCapabilityProbe(CasOperation & op, const String & probe_prefix); } diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.cpp index 8bcab7242eee0..7df3b208db273 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.cpp @@ -641,6 +641,18 @@ std::vector CasOperation::removeManyByConditional( const std::vector & removals, const Retry & policy) { const size_t limit = std::min(kBulkDeleteMaxKeys, owner.backend->conditionalBatchDeleteKeyLimit()); + return removeManyByConditionalImpl(removals, policy, limit); +} + +std::vector CasOperation::removeManyByConditionalForProbe( + const std::vector & removals, const Retry & policy) +{ + return removeManyByConditionalImpl(removals, policy, kBulkDeleteMaxKeys); +} + +std::vector CasOperation::removeManyByConditionalImpl( + const std::vector & removals, const Retry & policy, size_t limit) +{ if (removals.size() > limit) throw Exception(ErrorCodes::LOGICAL_ERROR, "CAS removeManyByConditional: {} keys in one chunk, the limit is {}; the consumer chunks its input", diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.h index 08f2d3d881fa9..0ae68b1bfcf89 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.h @@ -33,6 +33,10 @@ namespace DB::ErrorCodes namespace DB::Cas { +class CasOperation; +struct CapabilityProbeResult; +CapabilityProbeResult runCapabilityProbe(CasOperation & op, const String & probe_prefix); + /// TRUE when the store's own answer proves this write never applied: a malformed request, an entity /// too large, an access denial, or a credential failure. Whether a STALE CREDENTIAL explains it is not /// asked here -- the engine asks the backend for fresh credentials first, and refuses only when no @@ -356,6 +360,13 @@ class CasOperation WriteResult readModifyWriteOnPresence(const String & key, const DecideOnMeta & decide, const Retry & policy); private: + friend CapabilityProbeResult runCapabilityProbe(CasOperation & op, const String & probe_prefix); + + std::vector removeManyByConditionalForProbe( + const std::vector & removals, const Retry & policy); + std::vector removeManyByConditionalImpl( + const std::vector & removals, const Retry & policy, size_t limit); + friend class CasRequests; /// The lane waits on this operation's own admission and reads and writes through its verbs; it /// needs the gate, the reservation, the resolve read and the give-up helpers, never the transport. diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPool.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPool.cpp index 8fde07d1e9839..3057f9d1cac50 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPool.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPool.cpp @@ -523,7 +523,9 @@ PoolPtr Pool::open(BackendPtr backend, PoolConfig config) /// namespace only (never the content planes) — acceptable. const UInt128 probe_uid = (static_cast(thread_local_rng()) << 64) | thread_local_rng(); CasOperation probe_op = bootstrap_requests.admit(); - runCapabilityProbe(probe_op, config.pool_prefix + "/_probe/" + u128ToHex(probe_uid)); + const CapabilityProbeResult probe_result = runCapabilityProbe( + probe_op, config.pool_prefix + "/_probe/" + u128ToHex(probe_uid)); + backend->pinConditionalBatchDeleteSupport(probe_result.conditional_batch_delete_supported); } else { diff --git a/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp b/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp index e277ebca41822..f7196beba4727 100644 --- a/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp +++ b/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp @@ -213,6 +213,7 @@ PoolPtr openPoolWithIoConcurrency( config.event_sink = std::move(event_sink); config.gc_io_pool_refuse_at_for_test = refuse_at_for_test; config.gc_bulk_delete_chunk_keys = gc_bulk_delete_chunk_keys; + config.skip_access_check = true; return Pool::open(std::move(backend), std::move(config)); } diff --git a/src/Disks/tests/gtest_cas_probe.cpp b/src/Disks/tests/gtest_cas_probe.cpp index 136f36370547b..3c2db3ee1348b 100644 --- a/src/Disks/tests/gtest_cas_probe.cpp +++ b/src/Disks/tests/gtest_cas_probe.cpp @@ -33,6 +33,41 @@ TEST(CASProbe, PassesOnEnforcingBackend) EXPECT_TRUE(op.list("p/.cas_probe", "", 10, Retry::once()).keys.empty()); // probe cleans up after itself } +namespace +{ + +class IgnoresConditionalBatchTokensBackend : public InMemoryBackend +{ +public: + std::vector removeManyByConditional( + const std::vector & removals, TransportAccess & access) override + { + std::vector results; + results.reserve(removals.size()); + for (const RawConditionalRemove & removal : removals) + { + const auto current = InMemoryBackend::head(removal.key, access); + results.push_back(current ? InMemoryBackend::remove(removal.key, current->value, access) : RawRemoval::Gone); + } + return results; + } +}; + +} + +TEST(CASProbe, ReportsConditionalBatchCapability) +{ + InMemoryBackend enforcing; + auto enforcing_requests = makeRequests(enforcing); + auto enforcing_op = enforcing_requests.admit(); + EXPECT_TRUE(runCapabilityProbe(enforcing_op, "p/.cas_probe").conditional_batch_delete_supported); + + IgnoresConditionalBatchTokensBackend ignores; + auto ignores_requests = makeRequests(ignores); + auto ignores_op = ignores_requests.admit(); + EXPECT_FALSE(runCapabilityProbe(ignores_op, "p/.cas_probe").conditional_batch_delete_supported); +} + TEST(CASProbe, FailsClosedOnNonEnforcingDelete) { auto b = std::make_shared(); @@ -100,6 +135,20 @@ TEST(CASProbe, ConcurrentMountsDoNotCollide) EXPECT_TRUE(probe_op.read("p/_probe/token", Retry::once()).has_value()); } +TEST(CASProbe, PoolPinsConditionalBatchCapabilityOnlyAfterProbe) +{ + auto backend = std::make_shared(); + EXPECT_EQ(backend->conditionalBatchDeleteKeyLimit(), 1u); + + auto store = Pool::open(backend, PoolConfig{.pool_prefix = "p", .server_root_id = "test"}); + EXPECT_EQ(store->poolBackendPtr()->conditionalBatchDeleteKeyLimit(), kBulkDeleteMaxKeys); + + auto skipped_backend = std::make_shared(); + auto skipped = Pool::open(skipped_backend, PoolConfig{ + .pool_prefix = "skip", .server_root_id = "test", .skip_access_check = true}); + EXPECT_EQ(skipped->poolBackendPtr()->conditionalBatchDeleteKeyLimit(), 1u); +} + /// RFC cas-s3-timeout-retry-control: a Native-mode mount over an object storage that does not support /// the SingleAttempt retry profile must never silently proceed under the disk's default (~500-attempt) /// transparent retry policy — see Backend::checkConditionalWriteSingleAttemptSupport. This calls the @@ -235,7 +284,7 @@ TEST(CASProbe, ReorderedProbePassesOnAllThreeDialects) auto op = requests.admit(); EXPECT_NO_THROW(runCapabilityProbe(op, "p/.cas_probe")) << "dialect " << static_cast(dialect); EXPECT_TRUE(op.list("p/.cas_probe", "", 10, Retry::once()).keys.empty()) << "dialect " << static_cast(dialect); - EXPECT_EQ(b.write_reached, 4) << "dialect " << static_cast(dialect); - EXPECT_EQ(b.remove_reached, 2) << "dialect " << static_cast(dialect); + EXPECT_EQ(b.write_reached, 7) << "dialect " << static_cast(dialect); + EXPECT_EQ(b.remove_reached, 3) << "dialect " << static_cast(dialect); } } From e1deef0ac5850722cac27757de0b0a154631c8d9 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Mon, 12 Oct 2026 11:45:41 +0200 Subject: [PATCH 08/30] Document conditional CAS delete batching Signed-off-by: Konstantin Morozov --- .../cas/architecture/garbage-collection.md | 15 ++++++++++----- docs/en/antalya/cas/configuration.md | 2 +- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/docs/en/antalya/cas/architecture/garbage-collection.md b/docs/en/antalya/cas/architecture/garbage-collection.md index a06663cbd5c17..1f8cb59647241 100644 --- a/docs/en/antalya/cas/architecture/garbage-collection.md +++ b/docs/en/antalya/cas/architecture/garbage-collection.md @@ -52,7 +52,7 @@ follower or a deferred round execution returns before that commit. | 8 | `fold_ref_intake` | fold | `GET` every new ref-log record and every referenced manifest, extracting blob source edges | | 9 | `fold_reduce` | fold | Merge prior edges, new deltas and the parent's condemned rows: spare, condemn, graduate or redelete each candidate; compute `suppress_destructive` | | 10 | `fold_seal_write` | fold | Write the new fold seal once, write-once deterministic, adopting a byte-identical replay instead of rewriting it | -| 11 | `pending_deletes` | fold | The single blob-body delete site: exact-token delete of entries a *previous* round marked `delete_pending`, up to the redelete budget, plus the outcome-log writes | +| 11 | `pending_deletes` | fold | The single blob-body delete site: `HEAD` each entry a *previous* round marked `delete_pending`, then exact-token conditional delete chunks where the mounted backend proved that capability, plus the outcome-log writes | | 12 | `meta_pool_wait` | fold | Drain the bounded pool of async `.meta` condemn-marker writes queued during the fold | | 13 | `round_commit` | fold | Retention-prune old generations, then publish the single `gc/state` `CAS` that adopts the whole round | | 14 | `handoff_reclaim` | post-`CAS` | Reclaim a generation a ref moved off during this round, which the ordinary retention prune already skipped and will not revisit | @@ -401,8 +401,11 @@ forensic outcome logs. - **Reads:** one `HEAD` per `redelete` entry — the persisted condemned token cannot itself be a precondition, so the round observes the blob first, which also settles the absent case without spending a conditional delete -- **Writes / deletes:** a `DELETE` conditional on the observed etag for each `redelete` entry whose - live etag matches the condemned token; one write-once outcome log per shard with settled entries +- **Writes / deletes:** conditional delete chunks for entries whose live etag matches the condemned + token. A chunk is bounded by `cas_gc_bulk_delete_chunk_keys`, the backend conditional-batch limit, + and 1000 keys. An unprobed, unsupported, or `skip_access_check` mount uses one-key chunks. `.meta` + cleanup remains one per settled blob; it is not part of the conditional delete request. One + write-once outcome log is written per shard with settled entries - **Safety:** an entry is deletable only because a previously committed fold seal published it `delete_pending` — durable state from an earlier commit, safe at any leader staleness. Exact token means a stale leader cannot delete a fresh incarnation; `NotFound` and `TokenMismatch` are @@ -410,7 +413,9 @@ forensic outcome logs. - **Fails the round if:** a backend delete marker appears in a response — object versioning on a mis-provisioned pool (`LOGICAL_ERROR`) - **Observability:** phase row `pending_deletes`; metrics `deleted`, `absent`, `redeleted`, - `graduated`, `replaced`, `spared`, `outcome_logs_written` + `graduated`, `replaced`, `spared`, `outcome_logs_written`, `delete_batches`. `delete_batches` + counts logical conditional-delete chunks submitted by GC. `DiskS3DeleteObjects` counts physical S3 + delete attempts, including request retries Under `suppress_destructive` the `redelete` set is empty by construction, so nothing is deleted, and there is no graduation either (would-be graduates are carried unchanged); sparing and superseding @@ -805,7 +810,7 @@ the chunk size reject `0`: | `cas_manifest_sweep_delete_budget_keys` | 100 | orphan-manifest sweep `DELETE` budget per round (phases 9, 18) | | `cas_gc_round_sweep_namespace_budget` | 20 | namespaces whose protection view the sweep may build per page (phase 9) | | `cas_gc_round_sweep_recovery_op_budget` | 5000 | committed-tail ref-log reads the sweep's recovery walk may spend (phase 9) | -| `cas_gc_bulk_delete_chunk_keys` | 1000 | keys per batch `DELETE` request for write-once families (phases 15, 17); `1` to `1000` | +| `cas_gc_bulk_delete_chunk_keys` | 1000 | keys per batch `DELETE` request for blob redelete and write-once families (phases 11, 15, 17); `1` to `1000` | | `cas_gc_meta_pool_size` | 16 | bounded pool for condemn-marker writes (phase 12) | | `cas_gc_io_concurrency` | 16 | bounded pool for the fold's read-ahead of checkpoints, ref logs, manifest bodies and zero-candidate `HEAD`s (phases 8, 9), the graduation `.meta` reads (phase 9), the orphan-sweep planning reads (phase 9), the rebuild read-ahead and the `pending_deletes` `HEAD` + conditional `DELETE` fan-out (phase 11); other GC requests run on the round thread; `1` runs the covered requests sequentially. `cas_gc_read_concurrency` is rejected without an alias; use `cas_gc_io_concurrency` instead | diff --git a/docs/en/antalya/cas/configuration.md b/docs/en/antalya/cas/configuration.md index 80d150de2652d..e83dc3bc34511 100644 --- a/docs/en/antalya/cas/configuration.md +++ b/docs/en/antalya/cas/configuration.md @@ -107,7 +107,7 @@ entirely before release. Treat this table as a snapshot of the current build, no | `cas_part_folder_cache_max_entry_bytes` | 16 MiB | Oversized part-folder views bypass retention above this size | | `cas_manifest_decode_cache_bytes` | 128 MiB | Manifest decode cache byte budget (`0` disables) | | `cas_gc_meta_pool_size` | `16` | Bounded pool size for GC per-hash freshness-meta writes | -| `cas_gc_io_concurrency` | `16` | Bounded pool size for GC object-storage requests that run in parallel: the fold's read-ahead (checkpoints, ref logs, manifests, zero-candidate HEADs), the orphan-manifest sweep planning reads, the `SYSTEM CAS GC REBUILD` read-ahead, the `pending_deletes` blob `HEAD` + conditional `DELETE` fan-out, and the namespace janitor's batch deletes of dead ref logs and snapshots. Not covered: meta writes (`cas_gc_meta_pool_size`) and all other GC requests, which run on the round thread. `1` runs the covered requests sequentially. `cas_gc_read_concurrency` is rejected without an alias; use `cas_gc_io_concurrency` instead | +| `cas_gc_io_concurrency` | `16` | Bounded pool size for GC object-storage requests that run in parallel: the fold's read-ahead (checkpoints, ref logs, manifests, zero-candidate HEADs), the orphan-manifest sweep planning reads, the `SYSTEM CAS GC REBUILD` read-ahead, the `pending_deletes` blob `HEAD` + conditional `DELETE` chunks, and the namespace janitor's batch deletes of dead ref logs and snapshots. S3 multi-key conditional chunks are enabled only after the writable mount proves their per-object ETag semantics; unsupported storage and `skip_access_check` use one-key conditional deletes. Not covered: meta writes (`cas_gc_meta_pool_size`) and all other GC requests, which run on the round thread. `1` runs the covered requests sequentially. `cas_gc_read_concurrency` is rejected without an alias; use `cas_gc_io_concurrency` instead | | `cas_attempt_timeout_ms` | `5000` | Budget for one HTTP attempt of a writable Native mount's control-plane requests (read, head, list, remove, conditional write), at least 1. Together with the connect cap it forms the attempt envelope (`cas_attempt_timeout_ms + 2 × cap`; the cap is `cas_attempt_timeout_ms` itself when the disk's `connect_timeout_ms` is `0`, else `min(connect_timeout_ms, cas_attempt_timeout_ms)`) that the lease arithmetic reserves: one TCP connect and one TLS handshake under the cap each, send/receive bounded per socket operation by `cas_attempt_timeout_ms`. With background renewal the cadence check requires `cas_mount_renew_period_ms + 2 × envelope + cas_lease_safety_margin_ms < cas_mount_lease_ttl_ms`, which puts an effective ceiling on the frozen connect cap: under the defaults (TTL 30000, period 10000, margin 2000) the envelope must stay under 9000, so a disk `connect_timeout_ms` of 2000 ms or more refuses to open writable — lower the connect timeout or raise the TTL if you hit this. Blob body publication does not use this budget: its object-creating request is sent once under the disk's `request_timeout_ms`, with the connect timeout under the same cap | | `cas_lease_safety_margin_ms` | `2000` | Startup-only margin validated against the mount lease TTL: the attempt envelope + `cas_lease_safety_margin_ms` must be strictly less than the mount lease TTL, and `cas_mount_renew_period_ms` + 2 × envelope + `cas_lease_safety_margin_ms` too, or the disk refuses to open writable | | `cas_unsafe_remount_no_delay` | `0` | Reclaim a mount slot that carries this server's own uuid at once after a hard restart, without observing the slot's token for the lease TTL. Unsafe whenever two processes can hold the same `server_uuid` (a copied uuid file, a stalled predecessor). After such a reclaim the predecessor can still start conditional writes until its own cutoff (`confirmed deadline − cas_lease_safety_margin_ms − 2 × envelope`) or until its next renewal meets the token guard, and a request it already sent may still materialize later. That is not a data hazard: ref-log keys carry `(writer_epoch, sequence)` and creates are conditional, so two writers can never commit different bodies to one key, and recovery's epoch seal settles any straggler (recovery fails closed after 64 successive seal-create attempts displaced by newly materializing old-epoch transactions). The exposure is availability, not data. Intended for test stands and deployments that guarantee one process per uuid | From c3ca3b39da2f8f2dc7dcda05392ec2931515678b Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Mon, 12 Oct 2026 12:29:46 +0200 Subject: [PATCH 09/30] Test CAS conditional delete fallback on RustFS Signed-off-by: Konstantin Morozov --- .../__init__.py | 0 .../configs/storage_conf.xml | 29 +++++ .../test.py | 115 ++++++++++++++++++ 3 files changed, 144 insertions(+) create mode 100644 tests/integration/test_cas_gc_conditional_batch_delete/__init__.py create mode 100644 tests/integration/test_cas_gc_conditional_batch_delete/configs/storage_conf.xml create mode 100644 tests/integration/test_cas_gc_conditional_batch_delete/test.py diff --git a/tests/integration/test_cas_gc_conditional_batch_delete/__init__.py b/tests/integration/test_cas_gc_conditional_batch_delete/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/integration/test_cas_gc_conditional_batch_delete/configs/storage_conf.xml b/tests/integration/test_cas_gc_conditional_batch_delete/configs/storage_conf.xml new file mode 100644 index 0000000000000..efb13348faabc --- /dev/null +++ b/tests/integration/test_cas_gc_conditional_batch_delete/configs/storage_conf.xml @@ -0,0 +1,29 @@ + + + + + object_storage + s3 + cas + 30 + 10000 + itest-cas-gc-conditional-batch + http://rustfs1:11121/test/cas_gc_conditional_batch/ + clickhouse + clickhouse + 1 + 3600 + 2 + + + + + +
+ disk_cas_gc_conditional_batch +
+
+
+
+
+
diff --git a/tests/integration/test_cas_gc_conditional_batch_delete/test.py b/tests/integration/test_cas_gc_conditional_batch_delete/test.py new file mode 100644 index 0000000000000..32d9a72dd677f --- /dev/null +++ b/tests/integration/test_cas_gc_conditional_batch_delete/test.py @@ -0,0 +1,115 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +STORAGE_POLICY = "cas_gc_conditional_batch" +DISK_NAME = "disk_cas_gc_conditional_batch" +BLOBS_PREFIX = "cas_gc_conditional_batch/blobs/" +CHUNK_KEYS = 2 +NUM_INSERTS = 6 +MAX_GC_ROUNDS = 12 + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + cluster.add_instance( + "node", + main_configs=["configs/storage_conf.xml"], + with_rustfs=True, + stay_alive=True, + ) + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def list_blob_bodies(): + return { + o.object_name + for o in cluster.rustfs_client.list_objects( + cluster.rustfs_bucket, BLOBS_PREFIX, recursive=True + ) + if not o.object_name.endswith(".meta") + } + + +def latest_gc_round(node): + node.query("SYSTEM FLUSH LOGS") + return int( + node.query( + f"SELECT max(round) FROM system.cas_gc_log WHERE disk_name = '{DISK_NAME}'" + ).strip() + ) + + +def gc_pending_delete_rows(node, after_round): + node.query("SYSTEM FLUSH LOGS") + rows = ( + node.query( + "SELECT p.phase_metrics['redeleted'], p.phase_metrics['delete_batches'], " + "p.phase_metrics['jobs_failed'] " + "FROM system.cas_gc_log AS p INNER JOIN " + "(SELECT round_id FROM system.cas_gc_log " + f"WHERE disk_name = '{DISK_NAME}' AND event_type = 'Finish' " + f"AND round > {after_round}) AS f USING round_id " + f"WHERE p.disk_name = '{DISK_NAME}' AND p.event_type = 'Phase' " + "AND p.phase = 'pending_deletes' AND p.phase_metrics['redeleted'] > 0 " + "ORDER BY p.event_time_microseconds" + ) + .strip() + .splitlines() + ) + return [tuple(int(x) for x in row.split("\t")) for row in rows] + + +def gc_redeleted_total(node, after_round): + node.query("SYSTEM FLUSH LOGS") + return int( + node.query( + "SELECT sum(entries_redeleted) FROM system.cas_gc_log " + f"WHERE disk_name = '{DISK_NAME}' AND event_type = 'Finish' " + f"AND round > {after_round}" + ).strip() + ) + + +def test_rustfs_pending_blob_deletes_use_singleton_fallback(): + node = cluster.instances["node"] + table = "conditional_batch_redelete" + blobs_before = list_blob_bodies() + node.query( + f"CREATE TABLE {table} (k UInt64, v String) ENGINE = MergeTree ORDER BY k " + f"SETTINGS storage_policy = '{STORAGE_POLICY}'" + ) + node.query(f"SYSTEM STOP MERGES {table}") + for i in range(NUM_INSERTS): + node.query( + f"INSERT INTO {table} SELECT number + {i * 1000}, " + f"concat(toString(number), '-{i}') FROM numbers(1000) " + "SETTINGS max_insert_block_size = 1000" + ) + + created_blobs = list_blob_bodies() - blobs_before + assert len(created_blobs) > CHUNK_KEYS + first_round = latest_gc_round(node) + node.query(f"DROP TABLE {table} SYNC") + + rows = [] + remaining = created_blobs + for _ in range(MAX_GC_ROUNDS): + node.query("SYSTEM CAS GC RUN") + rows = gc_pending_delete_rows(node, first_round) + remaining = created_blobs & list_blob_bodies() + if not remaining and rows: + break + + assert not remaining, f"GC left blob bodies in RustFS: {sorted(remaining)}" + assert rows, "pending_deletes never reported completed blob redeletes" + for redeleted, batches, jobs_failed in rows: + assert jobs_failed == 0 + assert batches == redeleted + assert gc_redeleted_total(node, first_round) >= len(created_blobs) From 6065d60f0556739759ca93986bd7d2fadaf5fd0f Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Mon, 12 Oct 2026 12:30:03 +0200 Subject: [PATCH 10/30] Document conditional delete validation boundary Signed-off-by: Konstantin Morozov --- docs/en/antalya/cas/architecture/garbage-collection.md | 6 ++++++ docs/en/antalya/cas/bucket-requirements.md | 2 +- docs/en/operations/system-tables/cas_gc_log.md | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/en/antalya/cas/architecture/garbage-collection.md b/docs/en/antalya/cas/architecture/garbage-collection.md index 1f8cb59647241..12121ffe6e9e2 100644 --- a/docs/en/antalya/cas/architecture/garbage-collection.md +++ b/docs/en/antalya/cas/architecture/garbage-collection.md @@ -417,6 +417,12 @@ forensic outcome logs. counts logical conditional-delete chunks submitted by GC. `DiskS3DeleteObjects` counts physical S3 delete attempts, including request retries +The project RustFS integration backend supports exact-token singleton deletes but does not implement +AWS per-object `ETag` conditions in `DeleteObjects`. Its mount probe therefore keeps one-key chunks; +integration coverage verifies this fallback and physical blob reclamation. Positive AWS batching is +covered by deterministic GC and scripted S3 transport tests. A live AWS run is still required to +validate the production service boundary end to end. + Under `suppress_destructive` the `redelete` set is empty by construction, so nothing is deleted, and there is no graduation either (would-be graduates are carried unchanged); sparing and superseding still happen. An outcome log is written only for a shard that collected at least one budget-admitted diff --git a/docs/en/antalya/cas/bucket-requirements.md b/docs/en/antalya/cas/bucket-requirements.md index a42a6714998a4..f813fcfde2f28 100644 --- a/docs/en/antalya/cas/bucket-requirements.md +++ b/docs/en/antalya/cas/bucket-requirements.md @@ -137,7 +137,7 @@ establish acceptance of Google's multipart, native-copy, and exact-delete wire b | AWS S3 | ✓ | Native `ETag`-based conditional dialect for mutable objects and exact deletion; blob publication is unconditional | | Google Cloud Storage | implementation complete; release gate pending | Generation-token dialect for mutable objects/native-token `HEAD`/exact deletion, opted into via `http_client = gcs_hmac` or `gcp_oauth`; blob publication uses ordinary copy/multipart. Real credentialed GCS groups have not run yet | | Azure Blob Storage | probably | Azure's REST API documents the equivalent conditional headers, but ClickHouse's Azure object-storage backend does not yet wire up a `CAS` conditional dialect the way the S3 and GCS paths do — untested, not validated by the capability probe | -| Other S3-compatible stores | only with enforced conditional operations | The capability probe is the actual gate: a store that silently ignores `If-None-Match`/`If-Match` (accepting and applying the write regardless) fails the probe and is refused. `RustFS` passes the full battery and is used as the project's test backend; `Garage` was evaluated and rejected because it silently ignores conditional operations | +| Other S3-compatible stores | only with enforced conditional operations | The capability probe is the actual gate: a store that silently ignores `If-None-Match`/`If-Match` (accepting and applying the write regardless) fails the probe and is refused. `RustFS` passes the required singleton conditional-operation battery and is used as the project's test backend, but it does not implement AWS per-object `ETag` conditions in `DeleteObjects`; conditional GC deletion therefore stays on one-key chunks. `Garage` was evaluated and rejected because it silently ignores conditional operations | The full mechanics of the two dialects — how the backend detects which one a given endpoint speaks, what the capability probe actually checks, and how exact-token deletes map onto each provider's diff --git a/docs/en/operations/system-tables/cas_gc_log.md b/docs/en/operations/system-tables/cas_gc_log.md index 53309724de68c..60f7e432a03bb 100644 --- a/docs/en/operations/system-tables/cas_gc_log.md +++ b/docs/en/operations/system-tables/cas_gc_log.md @@ -81,7 +81,7 @@ The phases, in execution order: | `fold_ref_intake` | Read and fold every new ref log and the manifest bodies its edges name. | one `GET` per new log, one `GET` per manifest edge | | `fold_reduce` | The per-shard in-degree merge: condemn, spare, graduate. | prior-run streaming `GET`s, one `HEAD` per zero-transition candidate, run `PUT`s | | `fold_seal_write` | Publish the new fold seal. | one `PUT` | -| `pending_deletes` | The single content-delete site: exact-token deletes of previously published `delete_pending` entries, plus the outcome logs. `phase_metrics` carries `jobs_scheduled` (entries sent to the GC I/O pool; `0` for no candidates, sequential execution, singleton batches, or refusal before the first submission) and `jobs_failed`. | one `HEAD` and at most one `DELETE` per entry, one outcome-log `PUT` per shard with rows | +| `pending_deletes` | The single content-delete site: exact-token deletes of previously published `delete_pending` entries, plus the outcome logs. `phase_metrics` carries `jobs_scheduled` (chunks sent to the GC I/O pool; `0` for no candidates, sequential execution, or refusal before the first submission), `jobs_failed`, and `delete_batches` (logical conditional-delete chunks). Unsupported or unprobed storage uses one-key chunks. | one `HEAD` per entry, conditional delete chunks, one outcome-log `PUT` per shard with rows | | `meta_pool_wait` | Drain the round's per-hash freshness-meta writes. | none on this thread — see the caveat below | | `round_commit` | The generation-retention prune and the round's single `gc/state` compare-and-swap. | prune `LIST`s and deletes, one compare-and-swap | | `handoff_reclaim` | Wholesale-reclaim generations a moved run ref stranded below the retention cursor. | prefix `LIST`s and deletes | From eef7b65489c0395e4fca2afae5c21356f861e5b4 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Mon, 12 Oct 2026 14:33:47 +0200 Subject: [PATCH 11/30] Harden conditional batch delete failures Signed-off-by: Konstantin Morozov --- .../en/operations/system-tables/cas_gc_log.md | 2 +- .../ContentAddressed/Gc/CasGc.cpp | 75 ++++++++--- .../ObjectStorages/S3/S3ObjectStorage.cpp | 72 ++++++++++- .../gtest_cas_gc_redelete_concurrency.cpp | 21 +++ .../gtest_cas_s3_bulk_delete_fallback.cpp | 121 +++++++++++++++++- 5 files changed, 256 insertions(+), 35 deletions(-) diff --git a/docs/en/operations/system-tables/cas_gc_log.md b/docs/en/operations/system-tables/cas_gc_log.md index 60f7e432a03bb..6f75a3b043b66 100644 --- a/docs/en/operations/system-tables/cas_gc_log.md +++ b/docs/en/operations/system-tables/cas_gc_log.md @@ -81,7 +81,7 @@ The phases, in execution order: | `fold_ref_intake` | Read and fold every new ref log and the manifest bodies its edges name. | one `GET` per new log, one `GET` per manifest edge | | `fold_reduce` | The per-shard in-degree merge: condemn, spare, graduate. | prior-run streaming `GET`s, one `HEAD` per zero-transition candidate, run `PUT`s | | `fold_seal_write` | Publish the new fold seal. | one `PUT` | -| `pending_deletes` | The single content-delete site: exact-token deletes of previously published `delete_pending` entries, plus the outcome logs. `phase_metrics` carries `jobs_scheduled` (chunks sent to the GC I/O pool; `0` for no candidates, sequential execution, or refusal before the first submission), `jobs_failed`, and `delete_batches` (logical conditional-delete chunks). Unsupported or unprobed storage uses one-key chunks. | one `HEAD` per entry, conditional delete chunks, one outcome-log `PUT` per shard with rows | +| `pending_deletes` | The single content-delete site: exact-token deletes of previously published `delete_pending` entries, plus the outcome logs. `phase_metrics` carries `jobs_scheduled` (entries submitted to the parallel observation wave; `0` for no candidates or sequential execution), `jobs_failed`, and `delete_batches` (logical conditional-delete chunks). Unsupported or unprobed storage uses one-key chunks. | one `HEAD` per entry, conditional delete chunks, one outcome-log `PUT` per shard with rows | | `meta_pool_wait` | Drain the round's per-hash freshness-meta writes. | none on this thread — see the caveat below | | `round_commit` | The generation-retention prune and the round's single `gc/state` compare-and-swap. | prune `LIST`s and deletes, one compare-and-swap | | `handoff_reclaim` | Wholesale-reclaim generations a moved run ref stranded below the retention cursor. | prefix `LIST`s and deletes | diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp index df20d62954540..eb269455d2da6 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp @@ -743,25 +743,49 @@ void Gc::redeleteBlobs( kBulkDeleteMaxKeys); ThreadPoolCallbackRunnerLocal delete_runner(pool, ThreadName::CAS_GC_REDELETE); std::vector::Task>> delete_handles; - for (size_t begin = 0; begin < matched.size(); begin += chunk_keys) + delete_handles.reserve((matched.size() + chunk_keys - 1) / chunk_keys); + SCOPE_EXIT_SAFE({ ThreadPoolCallbackRunnerLocal::waitForAllToFinish(delete_handles); }); + std::vector> delete_indices; + delete_indices.reserve(delete_handles.capacity()); + std::vector delete_submitted(entries.size(), false); + std::optional delete_schedule_error; + size_t delete_begin = 0; + try { - const size_t end = std::min(matched.size(), begin + chunk_keys); - std::vector removals; - removals.reserve(end - begin); - for (size_t pos = begin; pos < end; ++pos) + for (; delete_begin < matched.size(); delete_begin += chunk_keys) { - const size_t index = matched[pos]; - removals.push_back({io_results[index].blob_key, *io_results[index].expected_token}); - } - delete_handles.emplace_back(delete_runner.enqueueAndGiveOwnership( - [begin, end, chunk_removals = std::move(removals), &matched, &io_results, pool_store = store, resumed_generation = op.generation()] + const size_t end = std::min(matched.size(), delete_begin + chunk_keys); + std::vector indices(matched.begin() + delete_begin, matched.begin() + end); + std::vector removals; + removals.reserve(indices.size()); + for (const size_t index : indices) + removals.push_back({io_results[index].blob_key, *io_results[index].expected_token}); + if (io_pool_refuse_at_for_test == entries.size() + delete_handles.size()) { - CasOperation job_op = pool_store->openRequests().resume(resumed_generation); - const auto results = job_op.removeManyByConditional(chunk_removals, Retry::standard()); - for (size_t pos = begin; pos < end; ++pos) - io_results[matched[pos]].del = results[pos - begin]; - })); - ++ctx.delete_batches; + io_pool_refuse_at_for_test.reset(); + throw Exception( + ErrorCodes::CANNOT_SCHEDULE_TASK, + "Injected CAS GC re-delete batch enqueue refusal at index {}", + delete_handles.size()); + } + auto handle = delete_runner.enqueueAndGiveOwnership( + [indices, chunk_removals = std::move(removals), results = &io_results, pool_store = store, resumed_generation = op.generation()] + { + CasOperation job_op = pool_store->openRequests().resume(resumed_generation); + const auto outcomes = job_op.removeManyByConditional(chunk_removals, Retry::standard()); + for (size_t pos = 0; pos < indices.size(); ++pos) + (*results)[indices[pos]].del = outcomes[pos]; + }); + delete_handles.emplace_back(std::move(handle)); + delete_indices.emplace_back(std::move(indices)); + for (const size_t index : delete_indices.back()) + delete_submitted[index] = true; + ++ctx.delete_batches; + } + } + catch (...) + { + delete_schedule_error = IndexedException{matched[delete_begin], std::current_exception()}; } ThreadPoolCallbackRunnerLocal::waitForAllToFinish(delete_handles); for (size_t chunk = 0; chunk < delete_handles.size(); ++chunk) @@ -772,10 +796,8 @@ void Gc::redeleteBlobs( } catch (...) { - const size_t begin = chunk * chunk_keys; - const size_t end = std::min(matched.size(), begin + chunk_keys); - for (size_t pos = begin; pos < end; ++pos) - io_errors[matched[pos]] = std::current_exception(); + for (const size_t index : delete_indices[chunk]) + io_errors[index] = std::current_exception(); } } @@ -795,13 +817,24 @@ void Gc::redeleteBlobs( if (!first_error || schedule_error->index < first_error->index) first_error = schedule_error; } + if (delete_schedule_error) + { + reportRedeleteFailure( + ctx, + entries[delete_schedule_error->index], + layout, + delete_schedule_error->exception, + matched.size() - delete_begin); + if (!first_error || delete_schedule_error->index < first_error->index) + first_error = delete_schedule_error; + } /// Every submitted delete of the batch has already run. If `applyRedeleteOutcome` throws, the batch stays deleted /// but unrecorded: the round fails before its outcome logs and `gc/state` commit, every entry stays /// `delete_pending`, and the next round records the deleted ones as `Absent`. for (size_t i = 0; i < scheduled; ++i) { - if (!io_errors[i]) + if (!io_errors[i] && (!io_results[i].expected_token || delete_submitted[i])) applyRedeleteOutcome(ctx, entries[i], io_results[i]); } diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp index c3ccda5912b03..9ec32b890f229 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp @@ -665,13 +665,46 @@ std::vector S3ObjectStorage::removeObjectsIfTokensMatch request_s3.SetDelete(std::move(delete_request)); if (request.attempt_number != 0) S3::setClickhouseAttemptNumber(request_s3, request.attempt_number); + auto blob_storage_log = BlobStorageLogWriter::create(disk_name); + const auto log_object = [&](const StoredObject & object, Int32 error_code, const String & error_message) + { + if (blob_storage_log) + blob_storage_log->addEvent( + BlobStorageLogElement::EventType::Delete, + uri.bucket, + object.remote_path, + object.local_path, + object.bytes_size, + 0, + error_code, + error_message); + }; + const auto log_all = [&](Int32 error_code, const String & error_message) + { + for (const auto & object : objects) + log_object(object.object, error_code, error_message); + }; const auto outcome = refreshAndRetryOnExpiredCredentials([&] { ProfileEvents::increment(ProfileEvents::DiskS3DeleteObjects); - return clientForRetryProfile(request)->DeleteObjects(request_s3); + auto current_outcome = clientForRetryProfile(request)->DeleteObjects(request_s3); + if (!current_outcome.IsSuccess()) + { + const auto & error = current_outcome.GetError(); + log_all(static_cast(error.GetErrorType()), error.GetMessage()); + if (error.GetExceptionName() == "InvalidRequest" || error.GetExceptionName() == "InvalidArgument" + || error.GetExceptionName() == "NotImplemented") + { + s3_capabilities.setIsBatchDeleteSupported(false); + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "{} does not support DeleteObjects", getName()); + } + throw S3Exception( + PreformattedMessage::create("{} while conditionally removing objects from S3", error.GetMessage()), + error.GetErrorType(), error.GetExceptionName()); + } + return current_outcome; }); - if (!outcome.IsSuccess()) - throw S3Exception(outcome.GetError().GetErrorType(), "{} while conditionally removing objects from S3", outcome.GetError().GetMessage()); + s3_capabilities.setIsBatchDeleteSupported(true); std::unordered_set expected_keys; expected_keys.reserve(objects.size()); @@ -684,40 +717,67 @@ std::vector S3ObjectStorage::removeObjectsIfTokensMatch { const String key = deleted.GetKey(); if (!expected_keys.contains(key) || !response_keys.insert(key).second) + { + log_all(static_cast(Aws::S3::S3Errors::UNKNOWN), "DeleteObjects response has unexpected or duplicate keys"); throw Exception(ErrorCodes::LOGICAL_ERROR, "S3 conditional DeleteObjects response has unexpected or duplicate key {}", key); + } } for (const auto & error : outcome.GetResult().GetErrors()) { const String key = error.GetKey(); if (!expected_keys.contains(key) || !response_keys.insert(key).second) + { + log_all(static_cast(Aws::S3::S3Errors::UNKNOWN), "DeleteObjects response has unexpected or duplicate keys"); throw Exception(ErrorCodes::LOGICAL_ERROR, "S3 conditional DeleteObjects response has unexpected or duplicate key {}", key); + } } if (response_keys.size() != objects.size()) + { + log_all(static_cast(Aws::S3::S3Errors::UNKNOWN), "DeleteObjects response omitted keys"); throw Exception(ErrorCodes::LOGICAL_ERROR, "S3 conditional DeleteObjects response omitted keys"); + } std::vector results; results.reserve(objects.size()); + std::optional first_error_code; + String first_error_name; + String first_error_message; + String first_error_key; for (const ConditionalRemoveObject & object : objects) { const auto deleted = std::find_if(outcome.GetResult().GetDeleted().begin(), outcome.GetResult().GetDeleted().end(), [&](const auto & entry) { return entry.GetKey() == object.object.remote_path; }); if (deleted != outcome.GetResult().GetDeleted().end()) { + log_object(object.object, 0, ""); results.push_back({ConditionalRemoveOutcome::Removed, deleted->GetDeleteMarker()}); continue; } const auto error = std::find_if(outcome.GetResult().GetErrors().begin(), outcome.GetResult().GetErrors().end(), [&](const auto & entry) { return entry.GetKey() == object.object.remote_path; }); if (error == outcome.GetResult().GetErrors().end()) + { + log_all(static_cast(Aws::S3::S3Errors::UNKNOWN), "DeleteObjects response omitted keys"); throw Exception(ErrorCodes::LOGICAL_ERROR, "S3 conditional DeleteObjects response omitted key {}", object.object.remote_path); + } const auto code = classifyDeleteObjectsErrorCode(error->GetCode()); + log_object(object.object, static_cast(code), error->GetMessage()); if (error->GetCode() == "PreconditionFailed") results.push_back({ConditionalRemoveOutcome::TokenMismatch, false}); - else if (S3::isNotFoundError(code)) + else if (error->GetCode() == "NoSuchKey") results.push_back({ConditionalRemoveOutcome::NotFound, false}); - else - throw S3Exception(code, "{} while conditionally removing object {} from S3", error->GetMessage(), error->GetKey()); + else if (!first_error_code) + { + first_error_code = code; + first_error_name = error->GetCode(); + first_error_message = error->GetMessage(); + first_error_key = error->GetKey(); + } } + if (first_error_code) + throw S3Exception( + PreformattedMessage::create("{} while conditionally removing object {} from S3", first_error_message, first_error_key), + *first_error_code, first_error_name); return results; } diff --git a/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp b/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp index f7196beba4727..efd3df5dc024b 100644 --- a/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp +++ b/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp @@ -403,6 +403,27 @@ TEST(CASGCRedeleteConcurrency, ConditionalDeleteBatchesAreRecorded) EXPECT_EQ(rows->total("delete_batches"), 3u); } +TEST(CASGCRedeleteConcurrency, DeleteWaveEnqueueRefusalWaitsForSubmittedChunks) +{ + auto backend = std::make_shared(); + backend->pinConditionalBatchDeleteSupport(true); + auto store = openPoolWithIoConcurrency(backend, 4, 1, 0, {}, kBlobs + 1, 2); + Gc gc(store, kGc); + publishThenDrop(*backend, store, gc); + ASSERT_NO_FATAL_FAILURE(driveUntilAllGraduated(*backend, gc, store, kReplacedBlob)); + + RoundReport progress; + EXPECT_THROW( + gc.runRegularRound({}, /*allow_steal*/ true, UniversePolicy::Authoritative, &progress), + DB::Exception); + store->renewWatermarkOnce(); + + EXPECT_EQ(progress.redeleted, 2u); + EXPECT_EQ(progress.redelete_failed, 1u); + EXPECT_EQ(runRegularRoundReclaiming(gc).redeleted, kBlobs); + EXPECT_TRUE(allBlobsAbsent(*backend, *store)); +} + TEST(CASGCRedeleteConcurrency, BlobDeletesActuallyOverlap) { auto backend = std::make_shared(); diff --git a/src/Disks/tests/gtest_cas_s3_bulk_delete_fallback.cpp b/src/Disks/tests/gtest_cas_s3_bulk_delete_fallback.cpp index 1a779be23f55c..1750b00b5bb19 100644 --- a/src/Disks/tests/gtest_cas_s3_bulk_delete_fallback.cpp +++ b/src/Disks/tests/gtest_cas_s3_bulk_delete_fallback.cpp @@ -80,10 +80,13 @@ class ScriptedS3Server std::lock_guard lock(owner.mutex); owner.methods_seen.push_back(request.getMethod()); } - /// `DeleteObjects` carries a request body (the XML `` payload); leaving it unread on a - /// keep-alive connection makes Poco parse those leftover bytes as the start of the NEXT - /// request once this handler returns, corrupting the very next `DeleteObject` this test expects. - request.stream().ignore(std::numeric_limits::max()); + const std::string body( + std::istreambuf_iterator(request.stream()), + std::istreambuf_iterator()); + { + std::lock_guard lock(owner.mutex); + owner.bodies_seen.push_back(body); + } owner.responder(request, response); } }; @@ -108,6 +111,7 @@ class ScriptedS3Server Responder responder; mutable std::mutex mutex; std::vector methods_seen; + std::vector bodies_seen; public: explicit ScriptedS3Server(Responder responder_) @@ -127,13 +131,19 @@ class ScriptedS3Server std::lock_guard lock(mutex); return static_cast(std::count(methods_seen.begin(), methods_seen.end(), method)); } + + std::string lastBody() const + { + std::lock_guard lock(mutex); + return bodies_seen.empty() ? std::string{} : bodies_seen.back(); + } }; void sendXml(Poco::Net::HTTPServerResponse & response, Poco::Net::HTTPResponse::HTTPStatus status, const std::string & body) { response.setContentType("application/xml"); response.setContentLength(body.size()); - response.setStatus(status); + response.setStatusAndReason(status); auto & out = response.send(); out << body; out.flush(); @@ -211,7 +221,10 @@ void sendConditionalDeleteObjectsResult(Poco::Net::HTTPServerResponse & response } std::shared_ptr makeStorageForTest( - const std::string & endpoint, const DB::S3Capabilities & capabilities, std::optional objects_chunk_size_to_delete = {}) + const std::string & endpoint, + const DB::S3Capabilities & capabilities, + std::optional objects_chunk_size_to_delete = {}, + const DB::S3ObjectStorage::S3CredentialsRefreshCallback & credentials_refresh_callback = {}) { DB::RemoteHostFilter remote_host_filter; DB::S3::PocoHTTPClientConfiguration cfg = DB::S3::ClientFactory::instance().createClientConfiguration( @@ -249,7 +262,7 @@ std::shared_ptr makeStorageForTest( return std::make_shared( std::move(client), std::move(settings), DB::S3::URI(endpoint + "/test-bucket/"), capabilities, - DB::ObjectStorageKeyGeneratorPtr{}, "disk"); + DB::ObjectStorageKeyGeneratorPtr{}, "disk", true, credentials_refresh_callback); } DB::ContextPtr contextForTest() @@ -324,6 +337,100 @@ TEST(CASS3ConditionalBatchDelete, ReturnsPerKeyOutcomesInInputOrder) EXPECT_EQ(results[0].outcome, DB::ConditionalRemoveOutcome::Removed); EXPECT_EQ(results[1].outcome, DB::ConditionalRemoveOutcome::TokenMismatch); EXPECT_EQ(server.countMethod("POST"), 1u); + EXPECT_NE(server.lastBody().find("first"), std::string::npos); + EXPECT_NE(server.lastBody().find("etag-first"), std::string::npos); + EXPECT_NE(server.lastBody().find("second"), std::string::npos); + EXPECT_NE(server.lastBody().find("etag-second"), std::string::npos); +} + +TEST(CASS3ConditionalBatchDelete, ConvertsExplicitUnsupportedToNotImplemented) +{ + (void)contextForTest(); + + ScriptedS3Server server([](const Poco::Net::HTTPServerRequest &, Poco::Net::HTTPServerResponse & response) + { + sendBatchNotImplemented(response); + }); + auto storage = makeStorageForTest(server.getUrl(), DB::S3Capabilities{}); + + expectNotImplemented([&] + { + storage->removeObjectsIfTokensMatch( + {{DB::StoredObject("first"), "etag-first"}, {DB::StoredObject("second"), "etag-second"}}, + DB::ObjectStorageControlRequest{}); + }); + EXPECT_EQ(storage->batchDeleteKeyLimit(), 1u); +} + +TEST(CASS3ConditionalBatchDelete, RefreshesExpiredCredentialsBeforeReturningFailure) +{ + (void)contextForTest(); + + std::atomic requests{0}; + ScriptedS3Server server([&](const Poco::Net::HTTPServerRequest &, Poco::Net::HTTPServerResponse & response) + { + if (requests.fetch_add(1) == 0) + { + sendSingleDeleteError(response, Poco::Net::HTTPResponse::HTTP_FORBIDDEN, "ExpiredToken", "expired"); + return; + } + sendConditionalDeleteObjectsResult(response); + }); + + std::atomic refreshes{0}; + auto storage = makeStorageForTest( + server.getUrl(), DB::S3Capabilities{}, {}, + [&]() -> std::unique_ptr + { + ++refreshes; + DB::RemoteHostFilter remote_host_filter; + DB::S3::PocoHTTPClientConfiguration cfg = DB::S3::ClientFactory::instance().createClientConfiguration( + "us-east-1", remote_host_filter, 100, + DB::S3::PocoHTTPClientConfiguration::RetryStrategy{.max_retries = 0}, + false, false, false, true, {}, {}); + cfg.endpointOverride = server.getUrl(); + cfg.s3_use_adaptive_timeouts = false; + return DB::S3::ClientFactory::instance().create( + cfg, + DB::S3::ClientSettings{ + .use_virtual_addressing = false, + .disable_checksum = false, + .gcs_issue_compose_request = false, + .is_s3express_bucket = false, + }, + "ACCESS_KEY_ID", "SECRET_ACCESS_KEY", "", {}, {}, DB::S3::CredentialsConfiguration{}); + }); + + const auto results = storage->removeObjectsIfTokensMatch( + {{DB::StoredObject("first"), "etag-first"}, {DB::StoredObject("second"), "etag-second"}}, + DB::ObjectStorageControlRequest{}); + + ASSERT_EQ(results.size(), 2u); + EXPECT_EQ(refreshes.load(), 1u); + EXPECT_EQ(requests.load(), 2u); +} + +TEST(CASS3ConditionalBatchDelete, DoesNotTreatMissingBucketAsMissingObject) +{ + (void)contextForTest(); + + ScriptedS3Server server([](const Poco::Net::HTTPServerRequest &, Poco::Net::HTTPServerResponse & response) + { + sendDeleteObjectsResult(response, {{"first", "NoSuchBucket"}, {"second", "PreconditionFailed"}}); + }); + auto storage = makeStorageForTest(server.getUrl(), DB::S3Capabilities{}); + + try + { + storage->removeObjectsIfTokensMatch( + {{DB::StoredObject("first"), "etag-first"}, {DB::StoredObject("second"), "etag-second"}}, + DB::ObjectStorageControlRequest{}); + FAIL() << "expected NoSuchBucket to remain a hard error"; + } + catch (const DB::S3Exception & e) + { + EXPECT_EQ(e.getExceptionName(), "NoSuchBucket"); + } } TEST(CASS3ConditionalBatchDelete, RejectsDuplicateResponseKey) From 887e9e2984c2165ff92bfe2c1d99e30fe8e7255b Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Mon, 12 Oct 2026 18:18:59 +0200 Subject: [PATCH 12/30] Keep per-blob redelete state in one struct Replace the parallel `io_results`, `io_errors` and `delete_submitted` vectors in `Gc::redeleteBlobs` with one `RedeleteBlobState` per entry. The observation result (`live_token`) is now separate from the delete result (`del`), so `performRedeleteIo` becomes `observeRetiredBlob`. Rename `applyRedeleteOutcome` to `recordRedeleteSuccess` and `reportRedeleteFailure` to `recordRedeleteFailure`: the delete has already run when they are called, and `Outcome` already names the persisted `GcOutcomes` log. Rename the test hook and test names to match. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Konstantin Morozov --- .../ContentAddressed/Gc/CasGc.cpp | 90 +++++++++---------- .../ContentAddressed/Gc/CasGc.h | 14 +-- .../ContentAddressed/Pool/CasPool.h | 2 +- src/Disks/tests/cas_gc_marker_test_support.h | 4 +- src/Disks/tests/gtest_cas_gc_meta_writer.cpp | 2 +- .../gtest_cas_gc_redelete_concurrency.cpp | 22 ++--- 6 files changed, 68 insertions(+), 66 deletions(-) diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp index eb269455d2da6..606d4e88ad191 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp @@ -574,34 +574,33 @@ size_t Gc::bulkDeleteChunkKeys() const return std::clamp(std::min(configured, storage_limit), 1, kBulkDeleteMaxKeys); } -Gc::RedeleteIo Gc::performRedeleteIo(const RetiredEntry & entry, const Layout & layout, CasOperation & op) +void Gc::observeRetiredBlob( + const RetiredEntry & entry, const Layout & layout, CasOperation & op, RedeleteBlobState & redelete_blob_state) { - RedeleteIo io; - io.blob_key = layout.blobKey(entry.ref); + redelete_blob_state.blob_key = layout.blobKey(entry.ref); /// The condemned incarnation is a PERSISTED pair and cannot itself be a precondition, so /// the round observes the blob and compares the two renderings. Observing first also /// settles the absent case without spending a conditional delete against a key that is /// already gone. - const std::optional observed = op.head(io.blob_key, Retry::standard()); - if (observed) - { - if (entry.token.matches(observed->etag)) - io.expected_token = observed->etag; - else - io.del = Removal::Mismatch; - } - return io; + const std::optional observed = op.head(redelete_blob_state.blob_key, Retry::standard()); + if (!observed) + redelete_blob_state.del = Removal::Gone; + else if (entry.token.matches(observed->etag)) + redelete_blob_state.live_token = observed->etag; + else + redelete_blob_state.del = Removal::Mismatch; } -void Gc::applyRedeleteOutcome(RedeleteRoundContext & ctx, const RetiredEntry & entry, const RedeleteIo & io) +void Gc::recordRedeleteSuccess(RedeleteRoundContext & ctx, const RetiredEntry & entry, const RedeleteBlobState & redelete_blob_state) { - if (const auto & hook = store->poolConfig().gc_redelete_apply_hook_for_test) + if (const auto & hook = store->poolConfig().gc_redelete_record_hook_for_test) hook(entry.ref); - const OutcomeKind outcome_kind = io.del == Removal::Removed ? OutcomeKind::Deleted - : io.del == Removal::Gone ? OutcomeKind::Absent - : OutcomeKind::Replaced; + const Removal del = *redelete_blob_state.del; + const OutcomeKind outcome_kind = del == Removal::Removed ? OutcomeKind::Deleted + : del == Removal::Gone ? OutcomeKind::Absent + : OutcomeKind::Replaced; OutcomeEntry outcome{.kind = entry.kind, .ref = entry.ref, .token = entry.token, .outcome = outcome_kind}; - const String del_outcome{removalName(io.del)}; + const String del_outcome{removalName(del)}; /// The single content-delete site is attributable per row. A mismatch (a writer recreated /// the incarnation) is terminal-OK: the fresh incarnation is a live object. EventEmitter{*store}.emit( @@ -615,9 +614,9 @@ void Gc::applyRedeleteOutcome(RedeleteRoundContext & ctx, const RetiredEntry & e e.gen = ctx.snap_generation; e.outcome = del_outcome; e.reason = "delete_pending published by a prior pass; exact-incarnation delete (pre-CAS)"; - e.detail = {{"condemn_round", std::to_string(entry.condemn_round)}, {"key", io.blob_key}}; + e.detail = {{"condemn_round", std::to_string(entry.condemn_round)}, {"key", redelete_blob_state.blob_key}}; }); - /// The audit row is observability only -- the delete in `performRedeleteIo` already executed + /// The audit row is observability only -- the delete in `redeleteBlobs` already executed /// regardless of this cap. Skipping it here bounds the per-shard `GcOutcomes` body without /// skipping or deferring any destructive work. if (ctx.round_work_budget.outcomeEntryAvailable()) @@ -632,7 +631,7 @@ void Gc::applyRedeleteOutcome(RedeleteRoundContext & ctx, const RetiredEntry & e /// republication path (`PartWriteTxn::ensureBlobPresent`) reconciles the meta back to Clean /// right after the publication; deleting it here would race that legitimate Clean write for /// no reason (the meta is advisory, but there is no reason to touch it on that path at all). - if (io.del == Removal::Removed || io.del == Removal::Gone) + if (del == Removal::Removed || del == Removal::Gone) { /// The round before this one, not `entry.condemn_round`: the graduation gate re-stamps a marker it /// refused at its own, later round. Deleting a marker is safe for writers that adopt only on `Clean`; @@ -641,7 +640,7 @@ void Gc::applyRedeleteOutcome(RedeleteRoundContext & ctx, const RetiredEntry & e } } -void Gc::reportRedeleteFailure( +void Gc::recordRedeleteFailure( RedeleteRoundContext & ctx, const RetiredEntry & entry, const Layout & layout, @@ -665,8 +664,7 @@ void Gc::redeleteBlobs( const Layout & layout, CasOperation & op) { - std::vector io_results(entries.size()); - std::vector io_errors(entries.size()); + std::vector redelete_blob_states(entries.size()); std::optional schedule_error; size_t scheduled = 0; if (concurrency <= 1 || entries.size() <= 1) @@ -675,11 +673,11 @@ void Gc::redeleteBlobs( { try { - io_results[i] = performRedeleteIo(entries[i], layout, op); + observeRetiredBlob(entries[i], layout, op, redelete_blob_states[i]); } catch (...) { - io_errors[i] = std::current_exception(); + redelete_blob_states[i].error = std::current_exception(); } } scheduled = entries.size(); @@ -702,14 +700,14 @@ void Gc::redeleteBlobs( throw Exception(ErrorCodes::CANNOT_SCHEDULE_TASK, "Injected CAS GC re-delete enqueue refusal at index {}", scheduled); } handles.emplace_back(runner.enqueueAndGiveOwnership( - [slot = &io_results[scheduled], + [redelete_blob_state = &redelete_blob_states[scheduled], entry = entries[scheduled], layout_ptr = &layout, pool_store = store, admitted_generation] { CasOperation job_op = pool_store->openRequests().resume(admitted_generation); - *slot = performRedeleteIo(entry, *layout_ptr, job_op); + observeRetiredBlob(entry, *layout_ptr, job_op, *redelete_blob_state); })); } } @@ -728,14 +726,14 @@ void Gc::redeleteBlobs( } catch (...) { - io_errors[i] = std::current_exception(); + redelete_blob_states[i].error = std::current_exception(); } } } std::vector matched; for (size_t i = 0; i < scheduled; ++i) - if (!io_errors[i] && io_results[i].expected_token) + if (!redelete_blob_states[i].error && redelete_blob_states[i].live_token) matched.push_back(i); const size_t chunk_keys = std::clamp( std::min(store->poolConfig().gc_bulk_delete_chunk_keys, store->poolBackendPtr()->conditionalBatchDeleteKeyLimit()), @@ -747,7 +745,6 @@ void Gc::redeleteBlobs( SCOPE_EXIT_SAFE({ ThreadPoolCallbackRunnerLocal::waitForAllToFinish(delete_handles); }); std::vector> delete_indices; delete_indices.reserve(delete_handles.capacity()); - std::vector delete_submitted(entries.size(), false); std::optional delete_schedule_error; size_t delete_begin = 0; try @@ -759,7 +756,7 @@ void Gc::redeleteBlobs( std::vector removals; removals.reserve(indices.size()); for (const size_t index : indices) - removals.push_back({io_results[index].blob_key, *io_results[index].expected_token}); + removals.push_back({redelete_blob_states[index].blob_key, *redelete_blob_states[index].live_token}); if (io_pool_refuse_at_for_test == entries.size() + delete_handles.size()) { io_pool_refuse_at_for_test.reset(); @@ -769,17 +766,19 @@ void Gc::redeleteBlobs( delete_handles.size()); } auto handle = delete_runner.enqueueAndGiveOwnership( - [indices, chunk_removals = std::move(removals), results = &io_results, pool_store = store, resumed_generation = op.generation()] + [indices, + chunk_removals = std::move(removals), + states = &redelete_blob_states, + pool_store = store, + resumed_generation = op.generation()] { CasOperation job_op = pool_store->openRequests().resume(resumed_generation); const auto outcomes = job_op.removeManyByConditional(chunk_removals, Retry::standard()); for (size_t pos = 0; pos < indices.size(); ++pos) - (*results)[indices[pos]].del = outcomes[pos]; + (*states)[indices[pos]].del = outcomes[pos]; }); delete_handles.emplace_back(std::move(handle)); delete_indices.emplace_back(std::move(indices)); - for (const size_t index : delete_indices.back()) - delete_submitted[index] = true; ++ctx.delete_batches; } } @@ -797,29 +796,30 @@ void Gc::redeleteBlobs( catch (...) { for (const size_t index : delete_indices[chunk]) - io_errors[index] = std::current_exception(); + redelete_blob_states[index].error = std::current_exception(); } } std::optional first_error; for (size_t i = 0; i < scheduled; ++i) { - if (!io_errors[i]) + const std::exception_ptr & error = redelete_blob_states[i].error; + if (!error) continue; - reportRedeleteFailure(ctx, entries[i], layout, io_errors[i]); + recordRedeleteFailure(ctx, entries[i], layout, error); if (!first_error) - first_error = IndexedException{i, io_errors[i]}; + first_error = IndexedException{i, error}; } if (schedule_error) { - reportRedeleteFailure( + recordRedeleteFailure( ctx, entries[schedule_error->index], layout, schedule_error->exception, entries.size() - schedule_error->index); if (!first_error || schedule_error->index < first_error->index) first_error = schedule_error; } if (delete_schedule_error) { - reportRedeleteFailure( + recordRedeleteFailure( ctx, entries[delete_schedule_error->index], layout, @@ -829,13 +829,13 @@ void Gc::redeleteBlobs( first_error = delete_schedule_error; } - /// Every submitted delete of the batch has already run. If `applyRedeleteOutcome` throws, the batch stays deleted + /// Every submitted delete of the batch has already run. If `recordRedeleteSuccess` throws, the batch stays deleted /// but unrecorded: the round fails before its outcome logs and `gc/state` commit, every entry stays /// `delete_pending`, and the next round records the deleted ones as `Absent`. for (size_t i = 0; i < scheduled; ++i) { - if (!io_errors[i] && (!io_results[i].expected_token || delete_submitted[i])) - applyRedeleteOutcome(ctx, entries[i], io_results[i]); + if (!redelete_blob_states[i].error && redelete_blob_states[i].del) + recordRedeleteSuccess(ctx, entries[i], redelete_blob_states[i]); } if (first_error) diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h index 080d7151bbc96..ab7ef140a9920 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h @@ -741,11 +741,12 @@ class Gc /// every destructive-work family the round touches — see `GcRoundWorkBudget`. GcRoundWorkBudget & work_budget); - struct RedeleteIo + struct RedeleteBlobState { String blob_key; - std::optional expected_token; - Removal del = Removal::Gone; + std::optional live_token; + std::optional del; + std::exception_ptr error; }; struct IndexedException @@ -766,11 +767,12 @@ class Gc uint64_t & delete_batches; }; - static RedeleteIo performRedeleteIo(const RetiredEntry & entry, const Layout & layout, CasOperation & op); + static void observeRetiredBlob( + const RetiredEntry & entry, const Layout & layout, CasOperation & op, RedeleteBlobState & redelete_blob_state); - void applyRedeleteOutcome(RedeleteRoundContext & ctx, const RetiredEntry & entry, const RedeleteIo & io); + void recordRedeleteSuccess(RedeleteRoundContext & ctx, const RetiredEntry & entry, const RedeleteBlobState & redelete_blob_state); - void reportRedeleteFailure( + void recordRedeleteFailure( RedeleteRoundContext & ctx, const RetiredEntry & entry, const Layout & layout, diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPool.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPool.h index 76335c42cd6f7..d664d0884fe9a 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPool.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPool.h @@ -215,7 +215,7 @@ struct PoolConfig std::function teardown_phase2_throw_for_test = {}; std::function teardown_phase3_throw_for_test = {}; - std::function gc_redelete_apply_hook_for_test = {}; + std::function gc_redelete_record_hook_for_test = {}; std::optional gc_io_pool_refuse_at_for_test = std::nullopt; /// Mount-lease TTL: how long a freshly-renewed mount lease is valid. The local diff --git a/src/Disks/tests/cas_gc_marker_test_support.h b/src/Disks/tests/cas_gc_marker_test_support.h index 0dbcbeb17baea..3f43697883398 100644 --- a/src/Disks/tests/cas_gc_marker_test_support.h +++ b/src/Disks/tests/cas_gc_marker_test_support.h @@ -46,13 +46,13 @@ inline BlobRef gateRef(const UInt128 & hash) /// A pool that folds every round, so a round's number is the number of rounds run. template PoolPtr openGatePool(std::shared_ptr backend, uint64_t io_concurrency = 1, uint64_t graduation_budget = 0, - std::function apply_hook = {}, uint64_t gc_shards = 1) + std::function record_hook = {}, uint64_t gc_shards = 1) { PoolConfig config{.pool_prefix = "p", .server_root_id = "test", .gc_fold_max_defer_rounds = 0, .gc_io_concurrency = io_concurrency}; config.gc_shards = gc_shards; config.gc_round_graduation_budget = graduation_budget; - config.gc_redelete_apply_hook_for_test = std::move(apply_hook); + config.gc_redelete_record_hook_for_test = std::move(record_hook); return Pool::open(std::move(backend), std::move(config)); } diff --git a/src/Disks/tests/gtest_cas_gc_meta_writer.cpp b/src/Disks/tests/gtest_cas_gc_meta_writer.cpp index 546bbfdf67116..fe762b0ff176e 100644 --- a/src/Disks/tests/gtest_cas_gc_meta_writer.cpp +++ b/src/Disks/tests/gtest_cas_gc_meta_writer.cpp @@ -178,7 +178,7 @@ TEST(CASGcMetaWriter, ThrowingRoundDrainsBeforeReturning) namespace { -/// The delete job, scheduled the way `Gc::applyRedeleteOutcome` schedules it, and drained. +/// The delete job, scheduled the way `Gc::recordRedeleteSuccess` schedules it, and drained. void runConfirmedMetaDelete(Gc & gc, const BlobRef & ref, uint64_t newest_round) { gc.metaWriterForTest().scheduleConfirmedMetaDelete(ref, newest_round); diff --git a/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp b/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp index efd3df5dc024b..4e9e24e903071 100644 --- a/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp +++ b/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp @@ -782,11 +782,11 @@ void expectBodiesPresentOnlyFor(Backend & backend, const Pool & store, const std EXPECT_EQ(blobAbsent(backend, store.layout(), blobHash(b)), !present.contains(b)) << "blob " << b; } -struct ApplyHookState +struct RecordHookState { std::atomic armed{false}; BlobRef target{}; - size_t applied = 0; + size_t recorded = 0; }; } @@ -1065,22 +1065,22 @@ TEST(CASGCRedeleteConcurrency, ShardedPoolDeletesEveryShard) EXPECT_EQ(shards_with_rows, expected_shards); } -TEST(CASGCRedeleteConcurrency, ApplyPhaseFaultLeavesEveryEntryForTheNextRound) +TEST(CASGCRedeleteConcurrency, RecordPhaseFaultLeavesEveryEntryForTheNextRound) { - auto state = std::make_shared(); + auto state = std::make_shared(); auto backend = std::make_shared(); PoolConfig config{.pool_prefix = "p", .server_root_id = "test"}; config.gc_io_concurrency = 4; - config.gc_redelete_apply_hook_for_test = [state](const BlobRef & ref) + config.gc_redelete_record_hook_for_test = [state](const BlobRef & ref) { if (!state->armed.load()) return; if (ref == state->target) { state->armed.store(false); - throw std::runtime_error("injected apply fault"); + throw std::runtime_error("injected record fault"); } - ++state->applied; + ++state->recorded; }; auto store = Pool::open(backend, std::move(config)); Gc gc(store, kGc); @@ -1102,10 +1102,10 @@ TEST(CASGCRedeleteConcurrency, ApplyPhaseFaultLeavesEveryEntryForTheNextRound) } store->renewWatermarkOnce(); - EXPECT_NE(message.find("injected apply fault"), String::npos) << message; - EXPECT_EQ(progress.redeleted, state->applied); - EXPECT_EQ(failedCounter() - failed_before, 0u) << "a failed apply is not a failed delete"; - EXPECT_TRUE(allBlobsAbsent(*backend, *store)) << "every delete finished before the apply walk started"; + EXPECT_NE(message.find("injected record fault"), String::npos) << message; + EXPECT_EQ(progress.redeleted, state->recorded); + EXPECT_EQ(failedCounter() - failed_before, 0u) << "a failed record is not a failed delete"; + EXPECT_TRUE(allBlobsAbsent(*backend, *store)) << "every delete finished before the record walk started"; const RoundReport next = runRegularRoundReclaiming(gc); EXPECT_EQ(next.redeleted, kBlobs); From 598fb7b3987d4010964128478f8b96ef11910fa4 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Mon, 12 Oct 2026 18:31:58 +0200 Subject: [PATCH 13/30] Extract conditional delete chunk preparation Move the selection of token-matched entries and their split into conditional delete chunks out of `Gc::redeleteBlobs` into `Gc::prepareConditionalDeleteChunks`. The second wave now only submits the prepared chunks, so `matched`, `delete_begin` and `delete_indices` are gone. The function and its value types are public so that the new `CASGCConditionalDeleteChunks` tests can call it without a pool or a backend request. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Konstantin Morozov --- .../ContentAddressed/Gc/CasGc.cpp | 68 ++++++++------- .../ContentAddressed/Gc/CasGc.h | 25 ++++-- .../gtest_cas_gc_redelete_concurrency.cpp | 86 +++++++++++++++++++ 3 files changed, 138 insertions(+), 41 deletions(-) diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp index 606d4e88ad191..077ef366d6bdc 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp @@ -591,6 +591,23 @@ void Gc::observeRetiredBlob( redelete_blob_state.del = Removal::Mismatch; } +std::vector Gc::prepareConditionalDeleteChunks( + const std::vector & redelete_blob_states, size_t chunk_keys) +{ + std::vector chunks; + for (size_t i = 0; i < redelete_blob_states.size(); ++i) + { + const RedeleteBlobState & redelete_blob_state = redelete_blob_states[i]; + if (redelete_blob_state.error || !redelete_blob_state.live_token) + continue; + if (chunks.empty() || chunks.back().indices.size() == chunk_keys) + chunks.emplace_back(); + chunks.back().indices.push_back(i); + chunks.back().removals.push_back({redelete_blob_state.blob_key, *redelete_blob_state.live_token}); + } + return chunks; +} + void Gc::recordRedeleteSuccess(RedeleteRoundContext & ctx, const RetiredEntry & entry, const RedeleteBlobState & redelete_blob_state) { if (const auto & hook = store->poolConfig().gc_redelete_record_hook_for_test) @@ -731,60 +748,46 @@ void Gc::redeleteBlobs( } } - std::vector matched; - for (size_t i = 0; i < scheduled; ++i) - if (!redelete_blob_states[i].error && redelete_blob_states[i].live_token) - matched.push_back(i); const size_t chunk_keys = std::clamp( std::min(store->poolConfig().gc_bulk_delete_chunk_keys, store->poolBackendPtr()->conditionalBatchDeleteKeyLimit()), 1, kBulkDeleteMaxKeys); + const std::vector delete_chunks = prepareConditionalDeleteChunks(redelete_blob_states, chunk_keys); ThreadPoolCallbackRunnerLocal delete_runner(pool, ThreadName::CAS_GC_REDELETE); std::vector::Task>> delete_handles; - delete_handles.reserve((matched.size() + chunk_keys - 1) / chunk_keys); + delete_handles.reserve(delete_chunks.size()); SCOPE_EXIT_SAFE({ ThreadPoolCallbackRunnerLocal::waitForAllToFinish(delete_handles); }); - std::vector> delete_indices; - delete_indices.reserve(delete_handles.capacity()); std::optional delete_schedule_error; - size_t delete_begin = 0; + size_t delete_scheduled = 0; try { - for (; delete_begin < matched.size(); delete_begin += chunk_keys) + for (; delete_scheduled < delete_chunks.size(); ++delete_scheduled) { - const size_t end = std::min(matched.size(), delete_begin + chunk_keys); - std::vector indices(matched.begin() + delete_begin, matched.begin() + end); - std::vector removals; - removals.reserve(indices.size()); - for (const size_t index : indices) - removals.push_back({redelete_blob_states[index].blob_key, *redelete_blob_states[index].live_token}); - if (io_pool_refuse_at_for_test == entries.size() + delete_handles.size()) + if (io_pool_refuse_at_for_test == entries.size() + delete_scheduled) { io_pool_refuse_at_for_test.reset(); throw Exception( ErrorCodes::CANNOT_SCHEDULE_TASK, "Injected CAS GC re-delete batch enqueue refusal at index {}", - delete_handles.size()); + delete_scheduled); } - auto handle = delete_runner.enqueueAndGiveOwnership( - [indices, - chunk_removals = std::move(removals), + delete_handles.emplace_back(delete_runner.enqueueAndGiveOwnership( + [chunk = &delete_chunks[delete_scheduled], states = &redelete_blob_states, pool_store = store, resumed_generation = op.generation()] { CasOperation job_op = pool_store->openRequests().resume(resumed_generation); - const auto outcomes = job_op.removeManyByConditional(chunk_removals, Retry::standard()); - for (size_t pos = 0; pos < indices.size(); ++pos) - (*states)[indices[pos]].del = outcomes[pos]; - }); - delete_handles.emplace_back(std::move(handle)); - delete_indices.emplace_back(std::move(indices)); + const auto outcomes = job_op.removeManyByConditional(chunk->removals, Retry::standard()); + for (size_t pos = 0; pos < chunk->indices.size(); ++pos) + (*states)[chunk->indices[pos]].del = outcomes[pos]; + })); ++ctx.delete_batches; } } catch (...) { - delete_schedule_error = IndexedException{matched[delete_begin], std::current_exception()}; + delete_schedule_error = IndexedException{delete_chunks[delete_scheduled].indices.front(), std::current_exception()}; } ThreadPoolCallbackRunnerLocal::waitForAllToFinish(delete_handles); for (size_t chunk = 0; chunk < delete_handles.size(); ++chunk) @@ -795,7 +798,7 @@ void Gc::redeleteBlobs( } catch (...) { - for (const size_t index : delete_indices[chunk]) + for (const size_t index : delete_chunks[chunk].indices) redelete_blob_states[index].error = std::current_exception(); } } @@ -819,12 +822,11 @@ void Gc::redeleteBlobs( } if (delete_schedule_error) { + size_t delete_unsent = 0; + for (size_t chunk = delete_scheduled; chunk < delete_chunks.size(); ++chunk) + delete_unsent += delete_chunks[chunk].indices.size(); recordRedeleteFailure( - ctx, - entries[delete_schedule_error->index], - layout, - delete_schedule_error->exception, - matched.size() - delete_begin); + ctx, entries[delete_schedule_error->index], layout, delete_schedule_error->exception, delete_unsent); if (!first_error || delete_schedule_error->index < first_error->index) first_error = delete_schedule_error; } diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h index ab7ef140a9920..5965fe1b90af3 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h @@ -541,6 +541,23 @@ class Gc /// (throwaway keys) remove non-content objects they own. Adding a second content-delete site is a /// protocol defect. + struct RedeleteBlobState + { + String blob_key; + std::optional live_token; + std::optional del; + std::exception_ptr error; + }; + + struct ConditionalDeleteChunk + { + std::vector indices; + std::vector removals; + }; + + static std::vector prepareConditionalDeleteChunks( + const std::vector & redelete_blob_states, size_t chunk_keys); + private: /// Lease acquire/renew/steal per the documented observation protocol. On success `state` holds the /// committed gc/state (with our lease) and `state_etag` the etag that write created. @@ -741,14 +758,6 @@ class Gc /// every destructive-work family the round touches — see `GcRoundWorkBudget`. GcRoundWorkBudget & work_budget); - struct RedeleteBlobState - { - String blob_key; - std::optional live_token; - std::optional del; - std::exception_ptr error; - }; - struct IndexedException { size_t index; diff --git a/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp b/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp index 4e9e24e903071..1284303443f2c 100644 --- a/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp +++ b/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp @@ -1239,3 +1239,89 @@ TEST(CASGCRedeleteConcurrency, RoundsWithoutRedeletesWriteNoOutcomeLog) ADD_FAILURE() << "a round without redeletes or spares wrote the outcome log " << key << " with " << rows.size() << " row(s)"; EXPECT_TRUE(logs.empty()); } + +namespace +{ + +Gc::RedeleteBlobState liveRedeleteBlobState(CasOperation & op, const String & key) +{ + Gc::RedeleteBlobState redelete_blob_state; + redelete_blob_state.blob_key = key; + redelete_blob_state.live_token = std::get(op.create(key, "v", Retry::once())).etag; + return redelete_blob_state; +} + +Gc::RedeleteBlobState settledRedeleteBlobState(const String & key, Removal del) +{ + Gc::RedeleteBlobState redelete_blob_state; + redelete_blob_state.blob_key = key; + redelete_blob_state.del = del; + return redelete_blob_state; +} + +std::vector> chunkIndices(const std::vector & chunks) +{ + std::vector> indices; + for (const auto & chunk : chunks) + indices.push_back(chunk.indices); + return indices; +} + +} + +TEST(CASGCConditionalDeleteChunks, OnlyLiveTokensWithoutErrorsAreChunked) +{ + auto backend = std::make_shared(); + OperationForTest op(backend); + std::vector redelete_blob_states; + redelete_blob_states.push_back(liveRedeleteBlobState(*op, "a")); + redelete_blob_states.push_back(settledRedeleteBlobState("gone", Removal::Gone)); + redelete_blob_states.push_back(liveRedeleteBlobState(*op, "b")); + redelete_blob_states.push_back(settledRedeleteBlobState("replaced", Removal::Mismatch)); + redelete_blob_states.push_back(liveRedeleteBlobState(*op, "failed")); + redelete_blob_states.back().error = std::make_exception_ptr(std::runtime_error("head failed")); + redelete_blob_states.emplace_back(); + redelete_blob_states.push_back(liveRedeleteBlobState(*op, "c")); + + const auto chunks = Gc::prepareConditionalDeleteChunks(redelete_blob_states, 10); + + ASSERT_EQ(chunkIndices(chunks), (std::vector>{{0, 2, 6}})); + const auto & removals = chunks.front().removals; + ASSERT_EQ(removals.size(), 3u); + for (size_t pos = 0; pos < removals.size(); ++pos) + { + const auto & redelete_blob_state = redelete_blob_states[chunks.front().indices[pos]]; + EXPECT_EQ(removals[pos].key, redelete_blob_state.blob_key); + EXPECT_EQ(removals[pos].expected_token, *redelete_blob_state.live_token); + } +} + +TEST(CASGCConditionalDeleteChunks, ChunksAreBoundedByChunkKeys) +{ + auto backend = std::make_shared(); + OperationForTest op(backend); + std::vector redelete_blob_states; + for (const String key : {"a", "b", "c", "d", "e"}) + redelete_blob_states.push_back(liveRedeleteBlobState(*op, key)); + + EXPECT_EQ( + chunkIndices(Gc::prepareConditionalDeleteChunks(redelete_blob_states, 2)), + (std::vector>{{0, 1}, {2, 3}, {4}})); + EXPECT_EQ( + chunkIndices(Gc::prepareConditionalDeleteChunks(redelete_blob_states, 1)), + (std::vector>{{0}, {1}, {2}, {3}, {4}})); + EXPECT_EQ( + chunkIndices(Gc::prepareConditionalDeleteChunks(redelete_blob_states, 5)), + (std::vector>{{0, 1, 2, 3, 4}})); +} + +TEST(CASGCConditionalDeleteChunks, NothingEligibleGivesNoChunks) +{ + std::vector redelete_blob_states; + redelete_blob_states.push_back(settledRedeleteBlobState("gone", Removal::Gone)); + redelete_blob_states.push_back(settledRedeleteBlobState("replaced", Removal::Mismatch)); + redelete_blob_states.emplace_back(); + + EXPECT_TRUE(Gc::prepareConditionalDeleteChunks(redelete_blob_states, 2).empty()); + EXPECT_TRUE(Gc::prepareConditionalDeleteChunks({}, 2).empty()); +} From af4d3d62bef9dff97cba4133dc249f57f0b66410 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Mon, 12 Oct 2026 18:34:13 +0200 Subject: [PATCH 14/30] Add `Gc::conditionalDeleteChunkKeys` Move the conditional delete chunk size calculation out of `Gc::redeleteBlobs` into a method next to `Gc::bulkDeleteChunkKeys`. No behavior change. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Konstantin Morozov --- .../MetadataStorages/ContentAddressed/Gc/CasGc.cpp | 14 +++++++++----- .../MetadataStorages/ContentAddressed/Gc/CasGc.h | 2 ++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp index 077ef366d6bdc..da31f916da4f9 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp @@ -574,6 +574,13 @@ size_t Gc::bulkDeleteChunkKeys() const return std::clamp(std::min(configured, storage_limit), 1, kBulkDeleteMaxKeys); } +size_t Gc::conditionalDeleteChunkKeys() const +{ + const uint64_t configured = store->poolConfig().gc_bulk_delete_chunk_keys; + const uint64_t storage_limit = store->poolBackendPtr()->conditionalBatchDeleteKeyLimit(); + return std::clamp(std::min(configured, storage_limit), 1, kBulkDeleteMaxKeys); +} + void Gc::observeRetiredBlob( const RetiredEntry & entry, const Layout & layout, CasOperation & op, RedeleteBlobState & redelete_blob_state) { @@ -748,11 +755,8 @@ void Gc::redeleteBlobs( } } - const size_t chunk_keys = std::clamp( - std::min(store->poolConfig().gc_bulk_delete_chunk_keys, store->poolBackendPtr()->conditionalBatchDeleteKeyLimit()), - 1, - kBulkDeleteMaxKeys); - const std::vector delete_chunks = prepareConditionalDeleteChunks(redelete_blob_states, chunk_keys); + const std::vector delete_chunks + = prepareConditionalDeleteChunks(redelete_blob_states, conditionalDeleteChunkKeys()); ThreadPoolCallbackRunnerLocal delete_runner(pool, ThreadName::CAS_GC_REDELETE); std::vector::Task>> delete_handles; delete_handles.reserve(delete_chunks.size()); diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h index 5965fe1b90af3..9b677efe251d0 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h @@ -510,6 +510,8 @@ class Gc /// limit, clamped to [1, `kBulkDeleteMaxKeys`] because an empty request would never advance. size_t bulkDeleteChunkKeys() const; + size_t conditionalDeleteChunkKeys() const; + void setRebuildEdgeBudgetForTest(uint64_t n) { rebuild_edge_budget_override = n; } /// TEST SEAM: disable the round's journal trim so a folded event stays in the journal From a26ea76d25185c9296e4ecbb3dc26cadf3235cbd Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Mon, 12 Oct 2026 18:42:05 +0200 Subject: [PATCH 15/30] Share the I/O job runner between redelete waves Add `Gc::runRedeleteIoJobs` and use it for both the observation wave and the conditional delete wave in `Gc::redeleteBlobs`. It runs jobs inline when `concurrency <= 1` or there is at most one job, and on the GC I/O pool otherwise. It returns per-job errors and the enqueue refusal. `jobs_scheduled` still counts only observation jobs submitted to the pool. The delete wave now also runs inline at `concurrency <= 1`, and both waves use one injected enqueue refusal message. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Konstantin Morozov --- .../ContentAddressed/Gc/CasGc.cpp | 191 +++++++++--------- .../ContentAddressed/Gc/CasGc.h | 16 ++ 2 files changed, 107 insertions(+), 100 deletions(-) diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp index da31f916da4f9..350c9619d70d9 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp @@ -680,135 +680,125 @@ void Gc::recordRedeleteFailure( unsent_count ? fmt::format("; {} re-delete entries were not submitted", *unsent_count) : String{}); } -void Gc::redeleteBlobs( - RedeleteRoundContext & ctx, - const std::vector & entries, +Gc::IoJobsResult Gc::runRedeleteIoJobs( ThreadPool & pool, size_t concurrency, - const Layout & layout, - CasOperation & op) + size_t count, + size_t refuse_offset, + CasOperation & op, + const std::function & job) { - std::vector redelete_blob_states(entries.size()); - std::optional schedule_error; - size_t scheduled = 0; - if (concurrency <= 1 || entries.size() <= 1) + IoJobsResult result; + result.errors.resize(count); + if (concurrency <= 1 || count <= 1) { - for (size_t i = 0; i < entries.size(); ++i) - { - try - { - observeRetiredBlob(entries[i], layout, op, redelete_blob_states[i]); - } - catch (...) - { - redelete_blob_states[i].error = std::current_exception(); - } - } - scheduled = entries.size(); - } - else - { - ThreadPoolCallbackRunnerLocal runner(pool, ThreadName::CAS_GC_REDELETE); - std::vector::Task>> handles; - handles.reserve(entries.size()); - SCOPE_EXIT_SAFE({ ThreadPoolCallbackRunnerLocal::waitForAllToFinish(handles); }); - - const uint64_t admitted_generation = op.generation(); - try - { - for (; scheduled < entries.size(); ++scheduled) - { - if (io_pool_refuse_at_for_test == scheduled) - { - io_pool_refuse_at_for_test.reset(); - throw Exception(ErrorCodes::CANNOT_SCHEDULE_TASK, "Injected CAS GC re-delete enqueue refusal at index {}", scheduled); - } - handles.emplace_back(runner.enqueueAndGiveOwnership( - [redelete_blob_state = &redelete_blob_states[scheduled], - entry = entries[scheduled], - layout_ptr = &layout, - pool_store = store, - admitted_generation] - { - CasOperation job_op = pool_store->openRequests().resume(admitted_generation); - observeRetiredBlob(entry, *layout_ptr, job_op, *redelete_blob_state); - })); - } - } - catch (...) - { - schedule_error = IndexedException{scheduled, std::current_exception()}; - } - ctx.jobs_scheduled += handles.size(); - - ThreadPoolCallbackRunnerLocal::waitForAllToFinish(handles); - for (size_t i = 0; i < handles.size(); ++i) + for (; result.scheduled < count; ++result.scheduled) { try { - handles[i]->future.get(); + job(result.scheduled, op); } catch (...) { - redelete_blob_states[i].error = std::current_exception(); + result.errors[result.scheduled] = std::current_exception(); } } + return result; } - const std::vector delete_chunks - = prepareConditionalDeleteChunks(redelete_blob_states, conditionalDeleteChunkKeys()); - ThreadPoolCallbackRunnerLocal delete_runner(pool, ThreadName::CAS_GC_REDELETE); - std::vector::Task>> delete_handles; - delete_handles.reserve(delete_chunks.size()); - SCOPE_EXIT_SAFE({ ThreadPoolCallbackRunnerLocal::waitForAllToFinish(delete_handles); }); - std::optional delete_schedule_error; - size_t delete_scheduled = 0; + result.on_pool = true; + ThreadPoolCallbackRunnerLocal runner(pool, ThreadName::CAS_GC_REDELETE); + std::vector::Task>> handles; + handles.reserve(count); + SCOPE_EXIT_SAFE({ ThreadPoolCallbackRunnerLocal::waitForAllToFinish(handles); }); + const uint64_t admitted_generation = op.generation(); try { - for (; delete_scheduled < delete_chunks.size(); ++delete_scheduled) + for (; result.scheduled < count; ++result.scheduled) { - if (io_pool_refuse_at_for_test == entries.size() + delete_scheduled) + if (io_pool_refuse_at_for_test == refuse_offset + result.scheduled) { io_pool_refuse_at_for_test.reset(); throw Exception( ErrorCodes::CANNOT_SCHEDULE_TASK, - "Injected CAS GC re-delete batch enqueue refusal at index {}", - delete_scheduled); + "Injected CAS GC re-delete enqueue refusal at index {}", + refuse_offset + result.scheduled); } - delete_handles.emplace_back(delete_runner.enqueueAndGiveOwnership( - [chunk = &delete_chunks[delete_scheduled], - states = &redelete_blob_states, - pool_store = store, - resumed_generation = op.generation()] + handles.emplace_back(runner.enqueueAndGiveOwnership( + [&job, index = result.scheduled, pool_store = store, admitted_generation] { - CasOperation job_op = pool_store->openRequests().resume(resumed_generation); - const auto outcomes = job_op.removeManyByConditional(chunk->removals, Retry::standard()); - for (size_t pos = 0; pos < chunk->indices.size(); ++pos) - (*states)[chunk->indices[pos]].del = outcomes[pos]; + CasOperation job_op = pool_store->openRequests().resume(admitted_generation); + job(index, job_op); })); - ++ctx.delete_batches; } } catch (...) { - delete_schedule_error = IndexedException{delete_chunks[delete_scheduled].indices.front(), std::current_exception()}; + result.schedule_error = IndexedException{result.scheduled, std::current_exception()}; } - ThreadPoolCallbackRunnerLocal::waitForAllToFinish(delete_handles); - for (size_t chunk = 0; chunk < delete_handles.size(); ++chunk) + + ThreadPoolCallbackRunnerLocal::waitForAllToFinish(handles); + for (size_t i = 0; i < handles.size(); ++i) { try { - delete_handles[chunk]->future.get(); + handles[i]->future.get(); } catch (...) { - for (const size_t index : delete_chunks[chunk].indices) - redelete_blob_states[index].error = std::current_exception(); + result.errors[i] = std::current_exception(); } } + return result; +} + +void Gc::redeleteBlobs( + RedeleteRoundContext & ctx, + const std::vector & entries, + ThreadPool & pool, + size_t concurrency, + const Layout & layout, + CasOperation & op) +{ + std::vector redelete_blob_states(entries.size()); + const IoJobsResult observe = runRedeleteIoJobs( + pool, + concurrency, + entries.size(), + 0, + op, + [&](size_t index, CasOperation & job_op) { observeRetiredBlob(entries[index], layout, job_op, redelete_blob_states[index]); }); + if (observe.on_pool) + ctx.jobs_scheduled += observe.scheduled; + for (size_t i = 0; i < observe.scheduled; ++i) + redelete_blob_states[i].error = observe.errors[i]; + + const std::vector delete_chunks + = prepareConditionalDeleteChunks(redelete_blob_states, conditionalDeleteChunkKeys()); + const IoJobsResult del = runRedeleteIoJobs( + pool, + concurrency, + delete_chunks.size(), + entries.size(), + op, + [&](size_t chunk, CasOperation & job_op) + { + const ConditionalDeleteChunk & delete_chunk = delete_chunks[chunk]; + const auto outcomes = job_op.removeManyByConditional(delete_chunk.removals, Retry::standard()); + for (size_t pos = 0; pos < delete_chunk.indices.size(); ++pos) + redelete_blob_states[delete_chunk.indices[pos]].del = outcomes[pos]; + }); + ctx.delete_batches += del.scheduled; + for (size_t chunk = 0; chunk < del.scheduled; ++chunk) + { + if (!del.errors[chunk]) + continue; + for (const size_t index : delete_chunks[chunk].indices) + redelete_blob_states[index].error = del.errors[chunk]; + } std::optional first_error; - for (size_t i = 0; i < scheduled; ++i) + for (size_t i = 0; i < observe.scheduled; ++i) { const std::exception_ptr & error = redelete_blob_states[i].error; if (!error) @@ -817,28 +807,29 @@ void Gc::redeleteBlobs( if (!first_error) first_error = IndexedException{i, error}; } - if (schedule_error) + if (observe.schedule_error) { + const IndexedException & schedule_error = *observe.schedule_error; recordRedeleteFailure( - ctx, entries[schedule_error->index], layout, schedule_error->exception, entries.size() - schedule_error->index); - if (!first_error || schedule_error->index < first_error->index) + ctx, entries[schedule_error.index], layout, schedule_error.exception, entries.size() - schedule_error.index); + if (!first_error || schedule_error.index < first_error->index) first_error = schedule_error; } - if (delete_schedule_error) + if (del.schedule_error) { + const size_t index = delete_chunks[del.schedule_error->index].indices.front(); size_t delete_unsent = 0; - for (size_t chunk = delete_scheduled; chunk < delete_chunks.size(); ++chunk) + for (size_t chunk = del.schedule_error->index; chunk < delete_chunks.size(); ++chunk) delete_unsent += delete_chunks[chunk].indices.size(); - recordRedeleteFailure( - ctx, entries[delete_schedule_error->index], layout, delete_schedule_error->exception, delete_unsent); - if (!first_error || delete_schedule_error->index < first_error->index) - first_error = delete_schedule_error; + recordRedeleteFailure(ctx, entries[index], layout, del.schedule_error->exception, delete_unsent); + if (!first_error || index < first_error->index) + first_error = IndexedException{index, del.schedule_error->exception}; } /// Every submitted delete of the batch has already run. If `recordRedeleteSuccess` throws, the batch stays deleted /// but unrecorded: the round fails before its outcome logs and `gc/state` commit, every entry stays /// `delete_pending`, and the next round records the deleted ones as `Absent`. - for (size_t i = 0; i < scheduled; ++i) + for (size_t i = 0; i < observe.scheduled; ++i) { if (!redelete_blob_states[i].error && redelete_blob_states[i].del) recordRedeleteSuccess(ctx, entries[i], redelete_blob_states[i]); diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h index 9b677efe251d0..f4b1a16781570 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h @@ -790,6 +790,22 @@ class Gc std::exception_ptr exception, std::optional unsent_count = std::nullopt); + struct IoJobsResult + { + bool on_pool = false; + size_t scheduled = 0; + std::vector errors; + std::optional schedule_error; + }; + + IoJobsResult runRedeleteIoJobs( + ThreadPool & pool, + size_t concurrency, + size_t count, + size_t refuse_offset, + CasOperation & op, + const std::function & job); + void redeleteBlobs( RedeleteRoundContext & ctx, const std::vector & entries, From 8b0a9b86eb2fcd923fcb4d5ef44dc67fd467a177 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Mon, 12 Oct 2026 18:50:12 +0200 Subject: [PATCH 16/30] Separate redelete result recording from `.meta` cleanup Split the tail of `Gc::redeleteBlobs`: - `Gc::recordRedeleteResults` records failures and successes and returns the error with the lowest entry index; - `Gc::scheduleBlobMetaDeletes` schedules `.meta` deletion for `Removed` and `Gone` blobs; - `Gc::redeleteBlobs` rethrows the error after both steps. `Gc::recordRedeleteSuccess` no longer schedules `.meta` deletion, so a later batch `.meta` delete changes only `Gc::scheduleBlobMetaDeletes`. If recording throws, no `.meta` deletion is scheduled for the round; the next round records the blobs as `Absent` and schedules it then. Rename `RedeleteBlobState::del` to `removal` (it can come from the observation wave, not only from a delete) and the wave results to `observe_jobs` and `delete_jobs`. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Konstantin Morozov --- .../ContentAddressed/Gc/CasGc.cpp | 120 +++++++++++------- .../ContentAddressed/Gc/CasGc.h | 21 ++- src/Disks/tests/gtest_cas_gc_meta_writer.cpp | 2 +- .../gtest_cas_gc_redelete_concurrency.cpp | 4 +- 4 files changed, 95 insertions(+), 52 deletions(-) diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp index 350c9619d70d9..ca01532055db6 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp @@ -591,11 +591,11 @@ void Gc::observeRetiredBlob( /// already gone. const std::optional observed = op.head(redelete_blob_state.blob_key, Retry::standard()); if (!observed) - redelete_blob_state.del = Removal::Gone; + redelete_blob_state.removal = Removal::Gone; else if (entry.token.matches(observed->etag)) redelete_blob_state.live_token = observed->etag; else - redelete_blob_state.del = Removal::Mismatch; + redelete_blob_state.removal = Removal::Mismatch; } std::vector Gc::prepareConditionalDeleteChunks( @@ -619,12 +619,12 @@ void Gc::recordRedeleteSuccess(RedeleteRoundContext & ctx, const RetiredEntry & { if (const auto & hook = store->poolConfig().gc_redelete_record_hook_for_test) hook(entry.ref); - const Removal del = *redelete_blob_state.del; - const OutcomeKind outcome_kind = del == Removal::Removed ? OutcomeKind::Deleted - : del == Removal::Gone ? OutcomeKind::Absent - : OutcomeKind::Replaced; + const Removal removal = *redelete_blob_state.removal; + const OutcomeKind outcome_kind = removal == Removal::Removed ? OutcomeKind::Deleted + : removal == Removal::Gone ? OutcomeKind::Absent + : OutcomeKind::Replaced; OutcomeEntry outcome{.kind = entry.kind, .ref = entry.ref, .token = entry.token, .outcome = outcome_kind}; - const String del_outcome{removalName(del)}; + const String removal_outcome{removalName(removal)}; /// The single content-delete site is attributable per row. A mismatch (a writer recreated /// the incarnation) is terminal-OK: the fresh incarnation is a live object. EventEmitter{*store}.emit( @@ -636,7 +636,7 @@ void Gc::recordRedeleteSuccess(RedeleteRoundContext & ctx, const RetiredEntry & e.token = renderIncarnation(entry.token); e.round = ctx.new_round; e.gen = ctx.snap_generation; - e.outcome = del_outcome; + e.outcome = removal_outcome; e.reason = "delete_pending published by a prior pass; exact-incarnation delete (pre-CAS)"; e.detail = {{"condemn_round", std::to_string(entry.condemn_round)}, {"key", redelete_blob_state.blob_key}}; }); @@ -650,17 +650,25 @@ void Gc::recordRedeleteSuccess(RedeleteRoundContext & ctx, const RetiredEntry & } ++ctx.report.redeleted; ProfileEvents::increment(ProfileEvents::CASGCRetiredRedeleted); +} + +void Gc::scheduleBlobMetaDeletes( + const std::vector & entries, + const std::vector & redelete_blob_states, + uint64_t newest_round) +{ /// Drop the per-hash meta only on a removal or a proven absence — a mismatch means a /// writer already published a fresh incarnation at this hash, and that writer's own /// republication path (`PartWriteTxn::ensureBlobPresent`) reconciles the meta back to Clean /// right after the publication; deleting it here would race that legitimate Clean write for /// no reason (the meta is advisory, but there is no reason to touch it on that path at all). - if (del == Removal::Removed || del == Removal::Gone) + for (size_t i = 0; i < redelete_blob_states.size(); ++i) { - /// The round before this one, not `entry.condemn_round`: the graduation gate re-stamps a marker it - /// refused at its own, later round. Deleting a marker is safe for writers that adopt only on `Clean`; - /// a 26.6.4 writer still backfills an absent marker and adopts. - meta_writer->scheduleConfirmedMetaDelete(entry.ref, ctx.new_round - 1); + const RedeleteBlobState & redelete_blob_state = redelete_blob_states[i]; + if (redelete_blob_state.error) + continue; + if (redelete_blob_state.removal == Removal::Removed || redelete_blob_state.removal == Removal::Gone) + meta_writer->scheduleConfirmedMetaDelete(entries[i].ref, newest_round); } } @@ -761,21 +769,21 @@ void Gc::redeleteBlobs( CasOperation & op) { std::vector redelete_blob_states(entries.size()); - const IoJobsResult observe = runRedeleteIoJobs( + const IoJobsResult observe_jobs = runRedeleteIoJobs( pool, concurrency, entries.size(), 0, op, [&](size_t index, CasOperation & job_op) { observeRetiredBlob(entries[index], layout, job_op, redelete_blob_states[index]); }); - if (observe.on_pool) - ctx.jobs_scheduled += observe.scheduled; - for (size_t i = 0; i < observe.scheduled; ++i) - redelete_blob_states[i].error = observe.errors[i]; + if (observe_jobs.on_pool) + ctx.jobs_scheduled += observe_jobs.scheduled; + for (size_t i = 0; i < observe_jobs.scheduled; ++i) + redelete_blob_states[i].error = observe_jobs.errors[i]; const std::vector delete_chunks = prepareConditionalDeleteChunks(redelete_blob_states, conditionalDeleteChunkKeys()); - const IoJobsResult del = runRedeleteIoJobs( + const IoJobsResult delete_jobs = runRedeleteIoJobs( pool, concurrency, delete_chunks.size(), @@ -786,57 +794,73 @@ void Gc::redeleteBlobs( const ConditionalDeleteChunk & delete_chunk = delete_chunks[chunk]; const auto outcomes = job_op.removeManyByConditional(delete_chunk.removals, Retry::standard()); for (size_t pos = 0; pos < delete_chunk.indices.size(); ++pos) - redelete_blob_states[delete_chunk.indices[pos]].del = outcomes[pos]; + redelete_blob_states[delete_chunk.indices[pos]].removal = outcomes[pos]; }); - ctx.delete_batches += del.scheduled; - for (size_t chunk = 0; chunk < del.scheduled; ++chunk) + ctx.delete_batches += delete_jobs.scheduled; + for (size_t chunk = 0; chunk < delete_jobs.scheduled; ++chunk) { - if (!del.errors[chunk]) + if (!delete_jobs.errors[chunk]) continue; for (const size_t index : delete_chunks[chunk].indices) - redelete_blob_states[index].error = del.errors[chunk]; + redelete_blob_states[index].error = delete_jobs.errors[chunk]; } + std::vector schedule_failures; + if (observe_jobs.schedule_error) + schedule_failures.push_back({*observe_jobs.schedule_error, entries.size() - observe_jobs.schedule_error->index}); + if (delete_jobs.schedule_error) + { + size_t delete_unsent = 0; + for (size_t chunk = delete_jobs.schedule_error->index; chunk < delete_chunks.size(); ++chunk) + delete_unsent += delete_chunks[chunk].indices.size(); + schedule_failures.push_back( + {{delete_chunks[delete_jobs.schedule_error->index].indices.front(), delete_jobs.schedule_error->exception}, delete_unsent}); + } + const std::optional first_error + = recordRedeleteResults(ctx, entries, layout, redelete_blob_states, observe_jobs.scheduled, schedule_failures); + scheduleBlobMetaDeletes(entries, redelete_blob_states, ctx.new_round - 1); + if (first_error) + std::rethrow_exception(first_error->exception); +} + +std::optional Gc::recordRedeleteResults( + RedeleteRoundContext & ctx, + const std::vector & entries, + const Layout & layout, + const std::vector & redelete_blob_states, + size_t observed, + const std::vector & schedule_failures) +{ std::optional first_error; - for (size_t i = 0; i < observe.scheduled; ++i) + auto keep_first_error = [&](const IndexedException & error) + { + if (!first_error || error.index < first_error->index) + first_error = error; + }; + + for (size_t i = 0; i < observed; ++i) { const std::exception_ptr & error = redelete_blob_states[i].error; if (!error) continue; recordRedeleteFailure(ctx, entries[i], layout, error); - if (!first_error) - first_error = IndexedException{i, error}; + keep_first_error({i, error}); } - if (observe.schedule_error) + for (const RedeleteScheduleFailure & failure : schedule_failures) { - const IndexedException & schedule_error = *observe.schedule_error; - recordRedeleteFailure( - ctx, entries[schedule_error.index], layout, schedule_error.exception, entries.size() - schedule_error.index); - if (!first_error || schedule_error.index < first_error->index) - first_error = schedule_error; - } - if (del.schedule_error) - { - const size_t index = delete_chunks[del.schedule_error->index].indices.front(); - size_t delete_unsent = 0; - for (size_t chunk = del.schedule_error->index; chunk < delete_chunks.size(); ++chunk) - delete_unsent += delete_chunks[chunk].indices.size(); - recordRedeleteFailure(ctx, entries[index], layout, del.schedule_error->exception, delete_unsent); - if (!first_error || index < first_error->index) - first_error = IndexedException{index, del.schedule_error->exception}; + recordRedeleteFailure(ctx, entries[failure.error.index], layout, failure.error.exception, failure.unsent); + keep_first_error(failure.error); } /// Every submitted delete of the batch has already run. If `recordRedeleteSuccess` throws, the batch stays deleted /// but unrecorded: the round fails before its outcome logs and `gc/state` commit, every entry stays /// `delete_pending`, and the next round records the deleted ones as `Absent`. - for (size_t i = 0; i < observe.scheduled; ++i) + for (size_t i = 0; i < observed; ++i) { - if (!redelete_blob_states[i].error && redelete_blob_states[i].del) + if (!redelete_blob_states[i].error && redelete_blob_states[i].removal) recordRedeleteSuccess(ctx, entries[i], redelete_blob_states[i]); } - - if (first_error) - std::rethrow_exception(first_error->exception); + return first_error; } RoundReport Gc::runRegularRound(std::function on_lease_acquired, bool allow_steal, UniversePolicy policy, diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h index f4b1a16781570..020bff6767c1b 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h @@ -547,7 +547,7 @@ class Gc { String blob_key; std::optional live_token; - std::optional del; + std::optional removal; std::exception_ptr error; }; @@ -766,6 +766,12 @@ class Gc std::exception_ptr exception; }; + struct RedeleteScheduleFailure + { + IndexedException error; + size_t unsent; + }; + struct RedeleteRoundContext { uint64_t new_round; @@ -790,6 +796,19 @@ class Gc std::exception_ptr exception, std::optional unsent_count = std::nullopt); + std::optional recordRedeleteResults( + RedeleteRoundContext & ctx, + const std::vector & entries, + const Layout & layout, + const std::vector & redelete_blob_states, + size_t observed, + const std::vector & schedule_failures); + + void scheduleBlobMetaDeletes( + const std::vector & entries, + const std::vector & redelete_blob_states, + uint64_t newest_round); + struct IoJobsResult { bool on_pool = false; diff --git a/src/Disks/tests/gtest_cas_gc_meta_writer.cpp b/src/Disks/tests/gtest_cas_gc_meta_writer.cpp index fe762b0ff176e..6663b2c154268 100644 --- a/src/Disks/tests/gtest_cas_gc_meta_writer.cpp +++ b/src/Disks/tests/gtest_cas_gc_meta_writer.cpp @@ -178,7 +178,7 @@ TEST(CASGcMetaWriter, ThrowingRoundDrainsBeforeReturning) namespace { -/// The delete job, scheduled the way `Gc::recordRedeleteSuccess` schedules it, and drained. +/// The delete job, scheduled the way `Gc::scheduleBlobMetaDeletes` schedules it, and drained. void runConfirmedMetaDelete(Gc & gc, const BlobRef & ref, uint64_t newest_round) { gc.metaWriterForTest().scheduleConfirmedMetaDelete(ref, newest_round); diff --git a/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp b/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp index 1284303443f2c..7646b88eb42a3 100644 --- a/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp +++ b/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp @@ -1251,11 +1251,11 @@ Gc::RedeleteBlobState liveRedeleteBlobState(CasOperation & op, const String & ke return redelete_blob_state; } -Gc::RedeleteBlobState settledRedeleteBlobState(const String & key, Removal del) +Gc::RedeleteBlobState settledRedeleteBlobState(const String & key, Removal removal) { Gc::RedeleteBlobState redelete_blob_state; redelete_blob_state.blob_key = key; - redelete_blob_state.del = del; + redelete_blob_state.removal = removal; return redelete_blob_state; } From d05e9bdde03c9e602fe485f8c33e46a63d287854 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Wed, 18 Nov 2026 12:59:07 +0100 Subject: [PATCH 17/30] Fix S3 attempt helper spelling after the 26.8 port The conditional batch-delete path still calls `setClickhouseAttemptNumber`, which was renamed to `setClickHouseAttemptNumber` in 26.8. Use the current helper so `S3ObjectStorage` compiles against its declaration. Related: https://github.com/Altinity/ClickHouse/pull/2534 CI: https://altinity-build-artifacts.s3.amazonaws.com/PRs/2534/ad2f86b097dcd86fd01a8cffeccb0f4e7ce52cfa/build_amd_debug/build_clickhouse/build_clickhouse.log Co-authored-by: Codex (GPT-6) Signed-off-by: Konstantin Morozov --- .../DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp index 95c803bcaaeba..3dd62dc7af4d7 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp @@ -667,7 +667,7 @@ std::vector S3ObjectStorage::removeObjectsIfTokensMatch request_s3.SetBucket(uri.bucket); request_s3.SetDelete(std::move(delete_request)); if (request.attempt_number != 0) - S3::setClickhouseAttemptNumber(request_s3, request.attempt_number); + S3::setClickHouseAttemptNumber(request_s3, request.attempt_number); auto blob_storage_log = BlobStorageLogWriter::create(disk_name); const auto log_object = [&](const StoredObject & object, Int32 error_code, const String & error_message) { From 9c2ebb72f215df8d4d7b492237f81fd1d1550567 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Wed, 18 Nov 2026 14:35:48 +0100 Subject: [PATCH 18/30] Clarify CAS GC conditional-delete job bookkeeping Name blob-state reads, chunk jobs, removal results and scheduling failures explicitly so per-blob state is distinct from per-chunk work. Build each `ConditionalDeleteChunk` locally and track an unfinished chunk with `std::optional` before moving it into the result. Add `const` to immutable value parameters and update the affected test helpers and references. Validation: `git diff --check` and static reference checks. Build and tests were not run for this change. Related: https://github.com/Altinity/ClickHouse/pull/2534 Co-authored-by: Codex (GPT-6) Signed-off-by: Konstantin Morozov --- .../Backend/CasInMemoryBackend.h | 2 +- .../Backend/CasInstrumentedBackend.h | 2 +- .../Backend/CasObjectStorageBackend.cpp | 2 +- .../ContentAddressed/Backend/CasRequests.cpp | 2 +- .../Backend/CasThrottlingBackend.h | 2 +- .../ContentAddressed/Gc/CasGc.cpp | 170 ++++++++++-------- .../ContentAddressed/Gc/CasGc.h | 29 +-- .../ObjectStorages/S3/S3ObjectStorage.cpp | 4 +- .../gtest_cas_gc_redelete_concurrency.cpp | 32 ++-- 9 files changed, 133 insertions(+), 112 deletions(-) diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInMemoryBackend.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInMemoryBackend.h index b7a279724390b..bd82f639145e9 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInMemoryBackend.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInMemoryBackend.h @@ -65,7 +65,7 @@ class InMemoryBackend : public Backend /// The emulation deletes keys one by one under its lock, so the CAS maximum is the only bound. size_t bulkDeleteKeyLimit() const override { return kBulkDeleteMaxKeys; } size_t conditionalBatchDeleteKeyLimit() const override { return conditional_batch_delete_supported ? kBulkDeleteMaxKeys : 1; } - void pinConditionalBatchDeleteSupport(bool supported) override { conditional_batch_delete_supported = supported; } + void pinConditionalBatchDeleteSupport(const bool supported) override { conditional_batch_delete_supported = supported; } /// Creates the key when `expected_value` is empty, or replaces the incarnation it names. A /// refused precondition leaves the store unchanged. Value enforcement can be disabled with diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInstrumentedBackend.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInstrumentedBackend.h index 041ad1d7b5198..0f21488d1cf32 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInstrumentedBackend.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasInstrumentedBackend.h @@ -165,7 +165,7 @@ class InstrumentedBackend final : public Backend uint64_t attemptEnvelopeMs() const override { return inner->attemptEnvelopeMs(); } size_t bulkDeleteKeyLimit() const override { return inner->bulkDeleteKeyLimit(); } size_t conditionalBatchDeleteKeyLimit() const override { return inner->conditionalBatchDeleteKeyLimit(); } - void pinConditionalBatchDeleteSupport(bool supported) override { inner->pinConditionalBatchDeleteSupport(supported); } + void pinConditionalBatchDeleteSupport(const bool supported) override { inner->pinConditionalBatchDeleteSupport(supported); } bool refreshCredentials() override { return inner->refreshCredentials(); } private: diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasObjectStorageBackend.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasObjectStorageBackend.cpp index 68bdb367d6e5c..674e3e90d930c 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasObjectStorageBackend.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasObjectStorageBackend.cpp @@ -1047,7 +1047,7 @@ size_t ObjectStorageBackend::conditionalBatchDeleteKeyLimit() const return object_storage->batchDeleteKeyLimit(); } -void ObjectStorageBackend::pinConditionalBatchDeleteSupport(bool supported) +void ObjectStorageBackend::pinConditionalBatchDeleteSupport(const bool supported) { conditional_batch_delete_supported.store(supported); } diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.cpp index 86de52bc854a9..70e0f48a58916 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.cpp @@ -664,7 +664,7 @@ std::vector CasOperation::removeManyByConditionalForProbe( } std::vector CasOperation::removeManyByConditionalImpl( - const std::vector & removals, const Retry & policy, size_t limit) + const std::vector & removals, const Retry & policy, const size_t limit) { if (removals.size() > limit) throw Exception(ErrorCodes::LOGICAL_ERROR, diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasThrottlingBackend.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasThrottlingBackend.h index 9789cb55c9f72..58a78ae85e4e8 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasThrottlingBackend.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasThrottlingBackend.h @@ -135,7 +135,7 @@ class ThrottlingBackend final : public Backend uint64_t attemptEnvelopeMs() const override { return inner->attemptEnvelopeMs(); } size_t bulkDeleteKeyLimit() const override { return inner->bulkDeleteKeyLimit(); } size_t conditionalBatchDeleteKeyLimit() const override { return inner->conditionalBatchDeleteKeyLimit(); } - void pinConditionalBatchDeleteSupport(bool supported) override { inner->pinConditionalBatchDeleteSupport(supported); } + void pinConditionalBatchDeleteSupport(const bool supported) override { inner->pinConditionalBatchDeleteSupport(supported); } bool refreshCredentials() override { return inner->refreshCredentials(); } void checkPoolPreconditions() override { inner->checkPoolPreconditions(); } void checkSkipAccessCheckSupport() override { inner->checkSkipAccessCheckSupport(); } diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp index cc098698d895d..6c7064e9db6bf 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp @@ -580,14 +580,14 @@ size_t Gc::bulkDeleteChunkKeys() const return std::clamp(std::min(configured, storage_limit), 1, kBulkDeleteMaxKeys); } -size_t Gc::conditionalDeleteChunkKeys() const +size_t Gc::conditionalDeleteKeysPerChunk() const { - const uint64_t configured = store->poolConfig().gc_bulk_delete_chunk_keys; + const uint64_t configured_limit = store->poolConfig().gc_bulk_delete_chunk_keys; const uint64_t storage_limit = store->poolBackendPtr()->conditionalBatchDeleteKeyLimit(); - return std::clamp(std::min(configured, storage_limit), 1, kBulkDeleteMaxKeys); + return std::clamp(std::min(configured_limit, storage_limit), 1, kBulkDeleteMaxKeys); } -void Gc::observeRetiredBlob( +void Gc::readRetiredBlobState( const RetiredEntry & entry, const Layout & layout, CasOperation & op, RedeleteBlobState & redelete_blob_state) { redelete_blob_state.blob_key = layout.blobKey(entry.ref); @@ -595,29 +595,38 @@ void Gc::observeRetiredBlob( /// the round observes the blob and compares the two renderings. Observing first also /// settles the absent case without spending a conditional delete against a key that is /// already gone. - const std::optional observed = op.head(redelete_blob_state.blob_key, Retry::standard()); - if (!observed) - redelete_blob_state.removal = Removal::Gone; - else if (entry.token.matches(observed->etag)) - redelete_blob_state.live_token = observed->etag; + const std::optional blob_data = op.head(redelete_blob_state.blob_key, Retry::standard()); + if (!blob_data) + redelete_blob_state.removal_result = Removal::Gone; + else if (entry.token.matches(blob_data->etag)) + redelete_blob_state.live_token = blob_data->etag; else - redelete_blob_state.removal = Removal::Mismatch; + redelete_blob_state.removal_result = Removal::Mismatch; } std::vector Gc::prepareConditionalDeleteChunks( - const std::vector & redelete_blob_states, size_t chunk_keys) + const std::vector & redelete_blob_states, const size_t max_keys_per_chunk) { std::vector chunks; + std::optional chunk; for (size_t i = 0; i < redelete_blob_states.size(); ++i) { const RedeleteBlobState & redelete_blob_state = redelete_blob_states[i]; if (redelete_blob_state.error || !redelete_blob_state.live_token) continue; - if (chunks.empty() || chunks.back().indices.size() == chunk_keys) - chunks.emplace_back(); - chunks.back().indices.push_back(i); - chunks.back().removals.push_back({redelete_blob_state.blob_key, *redelete_blob_state.live_token}); + + if (!chunk) + chunk.emplace(); + chunk->blob_state_indices.push_back(i); + chunk->conditional_removals.push_back({redelete_blob_state.blob_key, *redelete_blob_state.live_token}); + if (chunk->blob_state_indices.size() == max_keys_per_chunk) + { + chunks.push_back(std::move(*chunk)); + chunk.reset(); + } } + if (chunk) + chunks.push_back(std::move(*chunk)); return chunks; } @@ -625,12 +634,12 @@ void Gc::recordRedeleteSuccess(RedeleteRoundContext & ctx, const RetiredEntry & { if (const auto & hook = store->poolConfig().gc_redelete_record_hook_for_test) hook(entry.ref); - const Removal removal = *redelete_blob_state.removal; - const OutcomeKind outcome_kind = removal == Removal::Removed ? OutcomeKind::Deleted - : removal == Removal::Gone ? OutcomeKind::Absent - : OutcomeKind::Replaced; + const Removal removal_result = *redelete_blob_state.removal_result; + const OutcomeKind outcome_kind = removal_result == Removal::Removed ? OutcomeKind::Deleted + : removal_result == Removal::Gone ? OutcomeKind::Absent + : OutcomeKind::Replaced; OutcomeEntry outcome{.kind = entry.kind, .ref = entry.ref, .token = entry.token, .outcome = outcome_kind}; - const String removal_outcome{removalName(removal)}; + const String removal_outcome{removalName(removal_result)}; /// The single content-delete site is attributable per row. A mismatch (a writer recreated /// the incarnation) is terminal-OK: the fresh incarnation is a live object. EventEmitter{*store}.emit( @@ -659,7 +668,7 @@ void Gc::recordRedeleteSuccess(RedeleteRoundContext & ctx, const RetiredEntry & void Gc::scheduleBlobMetaDeletes( const std::vector & entries, const std::vector & redelete_blob_states, - uint64_t newest_round) + const uint64_t newest_round) { /// Drop the per-hash meta only on a removal or a proven absence — a mismatch means a /// writer already published a fresh incarnation at this hash, and that writer's own @@ -671,7 +680,7 @@ void Gc::scheduleBlobMetaDeletes( const RedeleteBlobState & redelete_blob_state = redelete_blob_states[i]; if (redelete_blob_state.error) continue; - if (redelete_blob_state.removal == Removal::Removed || redelete_blob_state.removal == Removal::Gone) + if (redelete_blob_state.removal_result == Removal::Removed || redelete_blob_state.removal_result == Removal::Gone) meta_writer->scheduleConfirmedMetaDelete(entries[i].ref, newest_round); } } @@ -680,8 +689,8 @@ void Gc::recordRedeleteFailure( RedeleteRoundContext & ctx, const RetiredEntry & entry, const Layout & layout, - std::exception_ptr exception, - std::optional unsent_count) + const std::exception_ptr exception, + const std::optional unscheduled_blob_count) { ProfileEvents::increment(ProfileEvents::CASGCRetiredRedeleteFailed); ++ctx.report.redelete_failed; @@ -689,22 +698,22 @@ void Gc::recordRedeleteFailure( "CAS gc: pending delete of blob {} (key `{}`, condemned at round {}) failed; the entry stays delete_pending " "and is retried in the next round: {}{}", blobIdOf(entry.ref), layout.blobKey(entry.ref), entry.condemn_round, getExceptionMessage(exception, false), - unsent_count ? fmt::format("; {} re-delete entries were not submitted", *unsent_count) : String{}); + unscheduled_blob_count ? fmt::format("; {} re-delete entries were not submitted", *unscheduled_blob_count) : String{}); } Gc::IoJobsResult Gc::runRedeleteIoJobs( ThreadPool & pool, - size_t concurrency, - size_t count, - size_t refuse_offset, + const size_t concurrency, + const size_t job_count, + const size_t refuse_offset, CasOperation & op, const std::function & job) { IoJobsResult result; - result.errors.resize(count); - if (concurrency <= 1 || count <= 1) + result.scheduled_job_errors.resize(job_count); + if (concurrency <= 1 || job_count <= 1) { - for (; result.scheduled < count; ++result.scheduled) + for (; result.scheduled < job_count; ++result.scheduled) { try { @@ -712,7 +721,7 @@ Gc::IoJobsResult Gc::runRedeleteIoJobs( } catch (...) { - result.errors[result.scheduled] = std::current_exception(); + result.scheduled_job_errors[result.scheduled] = std::current_exception(); } } return result; @@ -721,12 +730,14 @@ Gc::IoJobsResult Gc::runRedeleteIoJobs( result.on_pool = true; ThreadPoolCallbackRunnerLocal runner(pool, ThreadName::CAS_GC_REDELETE); std::vector::Task>> handles; - handles.reserve(count); + handles.reserve(job_count); + SCOPE_EXIT_SAFE({ ThreadPoolCallbackRunnerLocal::waitForAllToFinish(handles); }); + const uint64_t admitted_generation = op.generation(); try { - for (; result.scheduled < count; ++result.scheduled) + for (; result.scheduled < job_count; ++result.scheduled) { if (io_pool_refuse_at_for_test == refuse_offset + result.scheduled) { @@ -746,7 +757,7 @@ Gc::IoJobsResult Gc::runRedeleteIoJobs( } catch (...) { - result.schedule_error = IndexedException{result.scheduled, std::current_exception()}; + result.job_schedule_error = IndexedException{result.scheduled, std::current_exception()}; } ThreadPoolCallbackRunnerLocal::waitForAllToFinish(handles); @@ -758,7 +769,7 @@ Gc::IoJobsResult Gc::runRedeleteIoJobs( } catch (...) { - result.errors[i] = std::current_exception(); + result.scheduled_job_errors[i] = std::current_exception(); } } return result; @@ -768,61 +779,70 @@ void Gc::redeleteBlobs( RedeleteRoundContext & ctx, const std::vector & entries, ThreadPool & pool, - size_t concurrency, + const size_t concurrency, const Layout & layout, CasOperation & op) { std::vector redelete_blob_states(entries.size()); - const IoJobsResult observe_jobs = runRedeleteIoJobs( + const IoJobsResult read_blob_jobs = runRedeleteIoJobs( pool, concurrency, entries.size(), 0, op, - [&](size_t index, CasOperation & job_op) { observeRetiredBlob(entries[index], layout, job_op, redelete_blob_states[index]); }); - if (observe_jobs.on_pool) - ctx.jobs_scheduled += observe_jobs.scheduled; - for (size_t i = 0; i < observe_jobs.scheduled; ++i) - redelete_blob_states[i].error = observe_jobs.errors[i]; + [&](const size_t index, CasOperation & job_op) { readRetiredBlobState(entries[index], layout, job_op, redelete_blob_states[index]); }); + + if (read_blob_jobs.on_pool) + ctx.jobs_scheduled += read_blob_jobs.scheduled; + for (size_t i = 0; i < read_blob_jobs.scheduled; ++i) + redelete_blob_states[i].error = read_blob_jobs.scheduled_job_errors[i]; const std::vector delete_chunks - = prepareConditionalDeleteChunks(redelete_blob_states, conditionalDeleteChunkKeys()); - const IoJobsResult delete_jobs = runRedeleteIoJobs( + = prepareConditionalDeleteChunks(redelete_blob_states, conditionalDeleteKeysPerChunk()); + + const IoJobsResult delete_chunk_jobs = runRedeleteIoJobs( pool, concurrency, delete_chunks.size(), entries.size(), op, - [&](size_t chunk, CasOperation & job_op) + [&](const size_t chunk_index, CasOperation & job_op) { - const ConditionalDeleteChunk & delete_chunk = delete_chunks[chunk]; - const auto outcomes = job_op.removeManyByConditional(delete_chunk.removals, Retry::standard()); - for (size_t pos = 0; pos < delete_chunk.indices.size(); ++pos) - redelete_blob_states[delete_chunk.indices[pos]].removal = outcomes[pos]; + const ConditionalDeleteChunk & delete_chunk = delete_chunks[chunk_index]; + const auto removal_results = job_op.removeManyByConditional(delete_chunk.conditional_removals, Retry::standard()); + + for (size_t pos = 0; pos < delete_chunk.blob_state_indices.size(); ++pos) + redelete_blob_states[delete_chunk.blob_state_indices[pos]].removal_result = removal_results[pos]; }); - ctx.delete_batches += delete_jobs.scheduled; - for (size_t chunk = 0; chunk < delete_jobs.scheduled; ++chunk) + + ctx.delete_chunks += delete_chunk_jobs.scheduled; + for (size_t chunk_index = 0; chunk_index < delete_chunk_jobs.scheduled; ++chunk_index) { - if (!delete_jobs.errors[chunk]) + if (!delete_chunk_jobs.scheduled_job_errors[chunk_index]) continue; - for (const size_t index : delete_chunks[chunk].indices) - redelete_blob_states[index].error = delete_jobs.errors[chunk]; + for (const size_t index : delete_chunks[chunk_index].blob_state_indices) + redelete_blob_states[index].error = delete_chunk_jobs.scheduled_job_errors[chunk_index]; } - std::vector schedule_failures; - if (observe_jobs.schedule_error) - schedule_failures.push_back({*observe_jobs.schedule_error, entries.size() - observe_jobs.schedule_error->index}); - if (delete_jobs.schedule_error) + std::vector job_schedule_failures; + if (read_blob_jobs.job_schedule_error) + job_schedule_failures.push_back({*read_blob_jobs.job_schedule_error, entries.size() - read_blob_jobs.job_schedule_error->index}); + + if (delete_chunk_jobs.job_schedule_error) { - size_t delete_unsent = 0; - for (size_t chunk = delete_jobs.schedule_error->index; chunk < delete_chunks.size(); ++chunk) - delete_unsent += delete_chunks[chunk].indices.size(); - schedule_failures.push_back( - {{delete_chunks[delete_jobs.schedule_error->index].indices.front(), delete_jobs.schedule_error->exception}, delete_unsent}); + size_t unscheduled_delete_blob_count = 0; + for (size_t chunk_index = delete_chunk_jobs.job_schedule_error->index; chunk_index < delete_chunks.size(); ++chunk_index) + unscheduled_delete_blob_count += delete_chunks[chunk_index].blob_state_indices.size(); + job_schedule_failures.push_back( + {{delete_chunks[delete_chunk_jobs.job_schedule_error->index].blob_state_indices.front(), delete_chunk_jobs.job_schedule_error->exception}, + unscheduled_delete_blob_count}); } + const std::optional first_error - = recordRedeleteResults(ctx, entries, layout, redelete_blob_states, observe_jobs.scheduled, schedule_failures); + = recordRedeleteResults(ctx, entries, layout, redelete_blob_states, read_blob_jobs.scheduled, job_schedule_failures); + scheduleBlobMetaDeletes(entries, redelete_blob_states, ctx.new_round - 1); + if (first_error) std::rethrow_exception(first_error->exception); } @@ -832,8 +852,8 @@ std::optional Gc::recordRedeleteResults( const std::vector & entries, const Layout & layout, const std::vector & redelete_blob_states, - size_t observed, - const std::vector & schedule_failures) + const size_t scheduled_blob_head_job_count, + const std::vector & job_schedule_failures) { std::optional first_error; auto keep_first_error = [&](const IndexedException & error) @@ -842,7 +862,7 @@ std::optional Gc::recordRedeleteResults( first_error = error; }; - for (size_t i = 0; i < observed; ++i) + for (size_t i = 0; i < scheduled_blob_head_job_count; ++i) { const std::exception_ptr & error = redelete_blob_states[i].error; if (!error) @@ -850,18 +870,18 @@ std::optional Gc::recordRedeleteResults( recordRedeleteFailure(ctx, entries[i], layout, error); keep_first_error({i, error}); } - for (const RedeleteScheduleFailure & failure : schedule_failures) + for (const RedeleteScheduleFailure & failure : job_schedule_failures) { - recordRedeleteFailure(ctx, entries[failure.error.index], layout, failure.error.exception, failure.unsent); + recordRedeleteFailure(ctx, entries[failure.error.index], layout, failure.error.exception, failure.unscheduled_delete_blob_count); keep_first_error(failure.error); } /// Every submitted delete of the batch has already run. If `recordRedeleteSuccess` throws, the batch stays deleted /// but unrecorded: the round fails before its outcome logs and `gc/state` commit, every entry stays /// `delete_pending`, and the next round records the deleted ones as `Absent`. - for (size_t i = 0; i < observed; ++i) + for (size_t i = 0; i < scheduled_blob_head_job_count; ++i) { - if (!redelete_blob_states[i].error && redelete_blob_states[i].removal) + if (!redelete_blob_states[i].error && redelete_blob_states[i].removal_result) recordRedeleteSuccess(ctx, entries[i], redelete_blob_states[i]); } return first_error; @@ -1164,12 +1184,12 @@ RoundReport Gc::runRegularRound(std::function on_lease_acquired, bool al const uint64_t graduated_before = report.graduated; const uint64_t redelete_failed_before = report.redelete_failed; uint64_t redelete_jobs_scheduled = 0; - uint64_t redelete_delete_batches = 0; + uint64_t redelete_chunks = 0; const auto record_redelete_job_metrics = [&] { pending_deletes_timer->metric("jobs_scheduled", redelete_jobs_scheduled); pending_deletes_timer->metric("jobs_failed", report.redelete_failed - redelete_failed_before); - pending_deletes_timer->metric("delete_batches", redelete_delete_batches); + pending_deletes_timer->metric("delete_batches", redelete_chunks); }; SCOPE_EXIT_SAFE({ if (pending_deletes_timer) @@ -1206,7 +1226,7 @@ RoundReport Gc::runRegularRound(std::function on_lease_acquired, bool al .outcomes = outcomes, .outcome_tallies = outcome_tallies, .jobs_scheduled = redelete_jobs_scheduled, - .delete_batches = redelete_delete_batches}; + .delete_chunks = redelete_chunks}; redeleteBlobs(redelete_ctx, redelete_now, *io_pool, store->poolConfig().gc_io_concurrency, layout, op); for (const RetiredEntry & entry : merge.spared) { diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h index dee8b24b853b7..62d1cac4c931f 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h @@ -528,7 +528,7 @@ class Gc /// limit, clamped to [1, `kBulkDeleteMaxKeys`] because an empty request would never advance. size_t bulkDeleteChunkKeys() const; - size_t conditionalDeleteChunkKeys() const; + size_t conditionalDeleteKeysPerChunk() const; void setRebuildEdgeBudgetForTest(uint64_t n) { rebuild_edge_budget_override = n; } @@ -565,18 +565,19 @@ class Gc { String blob_key; std::optional live_token; - std::optional removal; + std::optional removal_result; std::exception_ptr error; }; struct ConditionalDeleteChunk { - std::vector indices; - std::vector removals; + /// `conditional_removals[i]` targets `redelete_blob_states[blob_state_indices[i]]`. + std::vector blob_state_indices; + std::vector conditional_removals; }; static std::vector prepareConditionalDeleteChunks( - const std::vector & redelete_blob_states, size_t chunk_keys); + const std::vector & redelete_blob_states, size_t max_keys_per_chunk); private: /// Lease acquire/renew/steal per the documented observation protocol. On success `state` holds the @@ -788,7 +789,7 @@ class Gc struct RedeleteScheduleFailure { IndexedException error; - size_t unsent; + size_t unscheduled_delete_blob_count{}; }; /// The settlement decisions of one shard in a round, counted where each is made: a decision whose @@ -814,10 +815,10 @@ class Gc std::map & outcomes; std::map & outcome_tallies; uint64_t & jobs_scheduled; - uint64_t & delete_batches; + uint64_t & delete_chunks; }; - static void observeRetiredBlob( + static void readRetiredBlobState( const RetiredEntry & entry, const Layout & layout, CasOperation & op, RedeleteBlobState & redelete_blob_state); void recordRedeleteSuccess(RedeleteRoundContext & ctx, const RetiredEntry & entry, const RedeleteBlobState & redelete_blob_state); @@ -827,15 +828,15 @@ class Gc const RetiredEntry & entry, const Layout & layout, std::exception_ptr exception, - std::optional unsent_count = std::nullopt); + std::optional unscheduled_blob_count = std::nullopt); std::optional recordRedeleteResults( RedeleteRoundContext & ctx, const std::vector & entries, const Layout & layout, const std::vector & redelete_blob_states, - size_t observed, - const std::vector & schedule_failures); + size_t scheduled_blob_head_job_count, + const std::vector & job_schedule_failures); void scheduleBlobMetaDeletes( const std::vector & entries, @@ -846,14 +847,14 @@ class Gc { bool on_pool = false; size_t scheduled = 0; - std::vector errors; - std::optional schedule_error; + std::vector scheduled_job_errors; + std::optional job_schedule_error; }; IoJobsResult runRedeleteIoJobs( ThreadPool & pool, size_t concurrency, - size_t count, + size_t job_count, size_t refuse_offset, CasOperation & op, const std::function & job); diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp index 3dd62dc7af4d7..2084e956f14ed 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp @@ -669,7 +669,7 @@ std::vector S3ObjectStorage::removeObjectsIfTokensMatch if (request.attempt_number != 0) S3::setClickHouseAttemptNumber(request_s3, request.attempt_number); auto blob_storage_log = BlobStorageLogWriter::create(disk_name); - const auto log_object = [&](const StoredObject & object, Int32 error_code, const String & error_message) + const auto log_object = [&](const StoredObject & object, const Int32 error_code, const String & error_message) { if (blob_storage_log) blob_storage_log->addEvent( @@ -682,7 +682,7 @@ std::vector S3ObjectStorage::removeObjectsIfTokensMatch error_code, error_message); }; - const auto log_all = [&](Int32 error_code, const String & error_message) + const auto log_all = [&](const Int32 error_code, const String & error_message) { for (const auto & object : objects) log_object(object.object, error_code, error_message); diff --git a/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp b/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp index 8c66bfe970964..9318f5fc3cc44 100644 --- a/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp +++ b/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp @@ -1251,20 +1251,20 @@ Gc::RedeleteBlobState liveRedeleteBlobState(CasOperation & op, const String & ke return redelete_blob_state; } -Gc::RedeleteBlobState settledRedeleteBlobState(const String & key, Removal removal) +Gc::RedeleteBlobState settledRedeleteBlobState(const String & key, const Removal removal_result) { Gc::RedeleteBlobState redelete_blob_state; redelete_blob_state.blob_key = key; - redelete_blob_state.removal = removal; + redelete_blob_state.removal_result = removal_result; return redelete_blob_state; } -std::vector> chunkIndices(const std::vector & chunks) +std::vector> chunkBlobStateIndices(const std::vector & chunks) { - std::vector> indices; + std::vector> blob_state_indices; for (const auto & chunk : chunks) - indices.push_back(chunk.indices); - return indices; + blob_state_indices.push_back(chunk.blob_state_indices); + return blob_state_indices; } } @@ -1285,14 +1285,14 @@ TEST(CASGCConditionalDeleteChunks, OnlyLiveTokensWithoutErrorsAreChunked) const auto chunks = Gc::prepareConditionalDeleteChunks(redelete_blob_states, 10); - ASSERT_EQ(chunkIndices(chunks), (std::vector>{{0, 2, 6}})); - const auto & removals = chunks.front().removals; - ASSERT_EQ(removals.size(), 3u); - for (size_t pos = 0; pos < removals.size(); ++pos) + ASSERT_EQ(chunkBlobStateIndices(chunks), (std::vector>{{0, 2, 6}})); + const auto & conditional_removals = chunks.front().conditional_removals; + ASSERT_EQ(conditional_removals.size(), 3u); + for (size_t pos = 0; pos < conditional_removals.size(); ++pos) { - const auto & redelete_blob_state = redelete_blob_states[chunks.front().indices[pos]]; - EXPECT_EQ(removals[pos].key, redelete_blob_state.blob_key); - EXPECT_EQ(removals[pos].expected_token, *redelete_blob_state.live_token); + const auto & redelete_blob_state = redelete_blob_states[chunks.front().blob_state_indices[pos]]; + EXPECT_EQ(conditional_removals[pos].key, redelete_blob_state.blob_key); + EXPECT_EQ(conditional_removals[pos].expected_token, *redelete_blob_state.live_token); } } @@ -1305,13 +1305,13 @@ TEST(CASGCConditionalDeleteChunks, ChunksAreBoundedByChunkKeys) redelete_blob_states.push_back(liveRedeleteBlobState(*op, key)); EXPECT_EQ( - chunkIndices(Gc::prepareConditionalDeleteChunks(redelete_blob_states, 2)), + chunkBlobStateIndices(Gc::prepareConditionalDeleteChunks(redelete_blob_states, 2)), (std::vector>{{0, 1}, {2, 3}, {4}})); EXPECT_EQ( - chunkIndices(Gc::prepareConditionalDeleteChunks(redelete_blob_states, 1)), + chunkBlobStateIndices(Gc::prepareConditionalDeleteChunks(redelete_blob_states, 1)), (std::vector>{{0}, {1}, {2}, {3}, {4}})); EXPECT_EQ( - chunkIndices(Gc::prepareConditionalDeleteChunks(redelete_blob_states, 5)), + chunkBlobStateIndices(Gc::prepareConditionalDeleteChunks(redelete_blob_states, 5)), (std::vector>{{0, 1, 2, 3, 4}})); } From dadc16a6f20b63d2a3026e25c91c1115fa4766f5 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Wed, 18 Nov 2026 15:57:11 +0100 Subject: [PATCH 19/30] Report CAS GC failures after each completed I/O wave Report blob-read failures before starting eligible deletes, and report chunk failures after all submitted deletes finish. Distinguish read, delete, and enqueue failures while preserving per-blob accounting and the original exception with the lowest input blob index. Cover mixed wave failures, warning order, and refused-enqueue tails in sequential and parallel execution. Co-authored-by: Codex (GPT-6) Signed-off-by: Konstantin Morozov --- .../cas/architecture/garbage-collection.md | 13 ++ .../en/operations/system-tables/cas_gc_log.md | 4 + .../ContentAddressed/Gc/CasGc.cpp | 129 +++++++++----- .../ContentAddressed/Gc/CasGc.h | 47 ++++-- .../gtest_cas_gc_redelete_concurrency.cpp | 157 ++++++++++++++++++ 5 files changed, 294 insertions(+), 56 deletions(-) diff --git a/docs/en/antalya/cas/architecture/garbage-collection.md b/docs/en/antalya/cas/architecture/garbage-collection.md index 8bef9c05befaf..99bc77d63bf4a 100644 --- a/docs/en/antalya/cas/architecture/garbage-collection.md +++ b/docs/en/antalya/cas/architecture/garbage-collection.md @@ -435,6 +435,19 @@ forensic outcome logs. counts logical conditional-delete chunks submitted by GC. `DiskS3DeleteObjects` counts physical S3 delete attempts, including request retries +GC reports read-job failures after all submitted blob-state reads finish, before starting the +conditional-delete wave. It reports delete-job failures after all submitted delete chunks finish. +Warnings distinguish read jobs, delete chunk jobs, and failures to enqueue either kind of job. +A failed delete job can have an unknown or partial storage outcome; its warning does not imply that +all objects in the chunk remain present. + +Read failures do not prevent deletion of successfully read siblings. After both waves, GC records +known outcomes and schedules eligible `.meta` cleanup, then rethrows the original job or enqueue +exception with the lowest input blob index. An unsuccessful round leaves the previously published +pending entries available to the next round. The existing `jobs_failed` metric and +`CASGCRetiredRedeleteFailed` event count each failed blob from an executed job, but count an enqueue +failure once per wave, regardless of the number of unsubmitted blobs reported in its warning. + The project RustFS integration backend supports exact-token singleton deletes but does not implement AWS per-object `ETag` conditions in `DeleteObjects`. Its mount probe therefore keeps one-key chunks; integration coverage verifies this fallback and physical blob reclamation. Positive AWS batching is diff --git a/docs/en/operations/system-tables/cas_gc_log.md b/docs/en/operations/system-tables/cas_gc_log.md index e42715aa0f11a..d08fcb3aaa9d4 100644 --- a/docs/en/operations/system-tables/cas_gc_log.md +++ b/docs/en/operations/system-tables/cas_gc_log.md @@ -133,6 +133,10 @@ Two caveats when reading these rows: `fold_ref_intake` and `fold_reduce` — also run on other threads and are missing from those phase rows' `ProfileEvents`. They still count in `system.events`. On `pending_deletes`, read `phase_metrics` `jobs_scheduled` / `jobs_failed` next to `redeleted`. + `jobs_failed` counts failed blobs from executed read/delete jobs, plus one per enqueue failure. + A failed chunk with multiple blobs therefore contributes multiple failures, while an unsubmitted + tail contributes one enqueue failure. GC warning messages distinguish these cases: read errors + are reported after the read wave completes, and delete errors after the delete wave completes. - Phase durations do not sum to the round's `duration_ms`. The round also performs untimed bookkeeping between phases, and the `Finish` row's `duration_ms` remains the authority on total round time. diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp index 6c7064e9db6bf..a3787767a5670 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp @@ -685,20 +685,34 @@ void Gc::scheduleBlobMetaDeletes( } } +std::string_view Gc::redeleteFailureName(const RedeleteFailureKind failure_kind) +{ + switch (failure_kind) + { + case RedeleteFailureKind::ReadBlobJob: return "read blob job"; + case RedeleteFailureKind::DeleteChunkJob: return "delete chunk job"; + case RedeleteFailureKind::ScheduleReadBlobJob: return "scheduling read blob job"; + case RedeleteFailureKind::ScheduleDeleteChunkJob: return "scheduling delete chunk job"; + } + UNREACHABLE(); +} + void Gc::recordRedeleteFailure( RedeleteRoundContext & ctx, const RetiredEntry & entry, const Layout & layout, + const RedeleteFailureKind failure_kind, const std::exception_ptr exception, const std::optional unscheduled_blob_count) { ProfileEvents::increment(ProfileEvents::CASGCRetiredRedeleteFailed); ++ctx.report.redelete_failed; LOG_WARNING(logger, - "CAS gc: pending delete of blob {} (key `{}`, condemned at round {}) failed; the entry stays delete_pending " + "CAS gc: {} failed for blob {} (key `{}`, condemned at round {}); the entry stays delete_pending " "and is retried in the next round: {}{}", - blobIdOf(entry.ref), layout.blobKey(entry.ref), entry.condemn_round, getExceptionMessage(exception, false), - unscheduled_blob_count ? fmt::format("; {} re-delete entries were not submitted", *unscheduled_blob_count) : String{}); + redeleteFailureName(failure_kind), blobIdOf(entry.ref), layout.blobKey(entry.ref), entry.condemn_round, getExceptionMessage(exception, false), + unscheduled_blob_count ? fmt::format("; {} blob {} were not submitted", *unscheduled_blob_count, + failure_kind == RedeleteFailureKind::ScheduleReadBlobJob ? "reads" : "deletions") : String{}); } Gc::IoJobsResult Gc::runRedeleteIoJobs( @@ -794,8 +808,8 @@ void Gc::redeleteBlobs( if (read_blob_jobs.on_pool) ctx.jobs_scheduled += read_blob_jobs.scheduled; - for (size_t i = 0; i < read_blob_jobs.scheduled; ++i) - redelete_blob_states[i].error = read_blob_jobs.scheduled_job_errors[i]; + std::optional first_error + = recordReadBlobJobFailures(ctx, entries, layout, redelete_blob_states, read_blob_jobs); const std::vector delete_chunks = prepareConditionalDeleteChunks(redelete_blob_states, conditionalDeleteKeysPerChunk()); @@ -816,75 +830,104 @@ void Gc::redeleteBlobs( }); ctx.delete_chunks += delete_chunk_jobs.scheduled; - for (size_t chunk_index = 0; chunk_index < delete_chunk_jobs.scheduled; ++chunk_index) - { - if (!delete_chunk_jobs.scheduled_job_errors[chunk_index]) - continue; - for (const size_t index : delete_chunks[chunk_index].blob_state_indices) - redelete_blob_states[index].error = delete_chunk_jobs.scheduled_job_errors[chunk_index]; - } - - std::vector job_schedule_failures; - if (read_blob_jobs.job_schedule_error) - job_schedule_failures.push_back({*read_blob_jobs.job_schedule_error, entries.size() - read_blob_jobs.job_schedule_error->index}); - if (delete_chunk_jobs.job_schedule_error) - { - size_t unscheduled_delete_blob_count = 0; - for (size_t chunk_index = delete_chunk_jobs.job_schedule_error->index; chunk_index < delete_chunks.size(); ++chunk_index) - unscheduled_delete_blob_count += delete_chunks[chunk_index].blob_state_indices.size(); - job_schedule_failures.push_back( - {{delete_chunks[delete_chunk_jobs.job_schedule_error->index].blob_state_indices.front(), delete_chunk_jobs.job_schedule_error->exception}, - unscheduled_delete_blob_count}); - } + const std::optional delete_error + = recordDeleteChunkJobFailures(ctx, entries, layout, redelete_blob_states, delete_chunks, delete_chunk_jobs); - const std::optional first_error - = recordRedeleteResults(ctx, entries, layout, redelete_blob_states, read_blob_jobs.scheduled, job_schedule_failures); + if (delete_error && (!first_error || delete_error->index < first_error->index)) + first_error = delete_error; + recordRedeleteSuccesses(ctx, entries, redelete_blob_states, read_blob_jobs.scheduled); scheduleBlobMetaDeletes(entries, redelete_blob_states, ctx.new_round - 1); if (first_error) std::rethrow_exception(first_error->exception); } -std::optional Gc::recordRedeleteResults( +std::optional Gc::recordReadBlobJobFailures( RedeleteRoundContext & ctx, const std::vector & entries, const Layout & layout, - const std::vector & redelete_blob_states, - const size_t scheduled_blob_head_job_count, - const std::vector & job_schedule_failures) + std::vector & redelete_blob_states, + const IoJobsResult & read_blob_jobs) { std::optional first_error; - auto keep_first_error = [&](const IndexedException & error) + for (size_t i = 0; i < read_blob_jobs.scheduled; ++i) + { + const std::exception_ptr & error = read_blob_jobs.scheduled_job_errors[i]; + redelete_blob_states[i].error = error; + if (!error) + continue; + recordRedeleteFailure(ctx, entries[i], layout, RedeleteFailureKind::ReadBlobJob, error); + if (!first_error || i < first_error->index) + first_error = IndexedException{i, error}; + } + if (read_blob_jobs.job_schedule_error) { + const IndexedException & error = *read_blob_jobs.job_schedule_error; + chassert(error.index < entries.size()); + recordRedeleteFailure(ctx, entries[error.index], layout, RedeleteFailureKind::ScheduleReadBlobJob, + error.exception, entries.size() - error.index); if (!first_error || error.index < first_error->index) first_error = error; - }; + } + return first_error; +} - for (size_t i = 0; i < scheduled_blob_head_job_count; ++i) +std::optional Gc::recordDeleteChunkJobFailures( + RedeleteRoundContext & ctx, + const std::vector & entries, + const Layout & layout, + std::vector & redelete_blob_states, + const std::vector & delete_chunks, + const IoJobsResult & delete_chunk_jobs) +{ + std::optional first_error; + for (size_t chunk_index = 0; chunk_index < delete_chunk_jobs.scheduled; ++chunk_index) { - const std::exception_ptr & error = redelete_blob_states[i].error; + const std::exception_ptr & error = delete_chunk_jobs.scheduled_job_errors[chunk_index]; if (!error) continue; - recordRedeleteFailure(ctx, entries[i], layout, error); - keep_first_error({i, error}); + for (const size_t index : delete_chunks[chunk_index].blob_state_indices) + { + redelete_blob_states[index].error = error; + recordRedeleteFailure(ctx, entries[index], layout, RedeleteFailureKind::DeleteChunkJob, error); + if (!first_error || index < first_error->index) + first_error = IndexedException{index, error}; + } } - for (const RedeleteScheduleFailure & failure : job_schedule_failures) + if (delete_chunk_jobs.job_schedule_error) { - recordRedeleteFailure(ctx, entries[failure.error.index], layout, failure.error.exception, failure.unscheduled_delete_blob_count); - keep_first_error(failure.error); + const size_t first_unscheduled_chunk_index = delete_chunk_jobs.job_schedule_error->index; + chassert(first_unscheduled_chunk_index < delete_chunks.size()); + size_t unscheduled_delete_blob_count = 0; + for (size_t chunk_index = first_unscheduled_chunk_index; chunk_index < delete_chunks.size(); ++chunk_index) + unscheduled_delete_blob_count += delete_chunks[chunk_index].blob_state_indices.size(); + + const size_t index = delete_chunks[first_unscheduled_chunk_index].blob_state_indices.front(); + const std::exception_ptr & error = delete_chunk_jobs.job_schedule_error->exception; + recordRedeleteFailure(ctx, entries[index], layout, RedeleteFailureKind::ScheduleDeleteChunkJob, + error, unscheduled_delete_blob_count); + if (!first_error || index < first_error->index) + first_error = IndexedException{index, error}; } + return first_error; +} +void Gc::recordRedeleteSuccesses( + RedeleteRoundContext & ctx, + const std::vector & entries, + const std::vector & redelete_blob_states, + const size_t scheduled_read_blob_job_count) +{ /// Every submitted delete of the batch has already run. If `recordRedeleteSuccess` throws, the batch stays deleted /// but unrecorded: the round fails before its outcome logs and `gc/state` commit, every entry stays /// `delete_pending`, and the next round records the deleted ones as `Absent`. - for (size_t i = 0; i < scheduled_blob_head_job_count; ++i) + for (size_t i = 0; i < scheduled_read_blob_job_count; ++i) { if (!redelete_blob_states[i].error && redelete_blob_states[i].removal_result) recordRedeleteSuccess(ctx, entries[i], redelete_blob_states[i]); } - return first_error; } RoundReport Gc::runRegularRound(std::function on_lease_acquired, bool allow_steal, UniversePolicy policy, @@ -1227,7 +1270,9 @@ RoundReport Gc::runRegularRound(std::function on_lease_acquired, bool al .outcome_tallies = outcome_tallies, .jobs_scheduled = redelete_jobs_scheduled, .delete_chunks = redelete_chunks}; + redeleteBlobs(redelete_ctx, redelete_now, *io_pool, store->poolConfig().gc_io_concurrency, layout, op); + for (const RetiredEntry & entry : merge.spared) { /// A fresh dedup-adopt raced the condemn (see the matching CasGcFold Debug log emitted diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h index 62d1cac4c931f..80178b9fed077 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -786,10 +787,22 @@ class Gc std::exception_ptr exception; }; - struct RedeleteScheduleFailure + enum class RedeleteFailureKind { - IndexedException error; - size_t unscheduled_delete_blob_count{}; + ReadBlobJob, + DeleteChunkJob, + ScheduleReadBlobJob, + ScheduleDeleteChunkJob, + }; + + static std::string_view redeleteFailureName(RedeleteFailureKind failure_kind); + + struct IoJobsResult + { + bool on_pool = false; + size_t scheduled = 0; + std::vector scheduled_job_errors; + std::optional job_schedule_error; }; /// The settlement decisions of one shard in a round, counted where each is made: a decision whose @@ -827,30 +840,36 @@ class Gc RedeleteRoundContext & ctx, const RetiredEntry & entry, const Layout & layout, + RedeleteFailureKind failure_kind, std::exception_ptr exception, std::optional unscheduled_blob_count = std::nullopt); - std::optional recordRedeleteResults( + std::optional recordReadBlobJobFailures( + RedeleteRoundContext & ctx, + const std::vector & entries, + const Layout & layout, + std::vector & redelete_blob_states, + const IoJobsResult & read_blob_jobs); + + std::optional recordDeleteChunkJobFailures( RedeleteRoundContext & ctx, const std::vector & entries, const Layout & layout, + std::vector & redelete_blob_states, + const std::vector & delete_chunks, + const IoJobsResult & delete_chunk_jobs); + + void recordRedeleteSuccesses( + RedeleteRoundContext & ctx, + const std::vector & entries, const std::vector & redelete_blob_states, - size_t scheduled_blob_head_job_count, - const std::vector & job_schedule_failures); + size_t scheduled_read_blob_job_count); void scheduleBlobMetaDeletes( const std::vector & entries, const std::vector & redelete_blob_states, uint64_t newest_round); - struct IoJobsResult - { - bool on_pool = false; - size_t scheduled = 0; - std::vector scheduled_job_errors; - std::optional job_schedule_error; - }; - IoJobsResult runRedeleteIoJobs( ThreadPool & pool, size_t concurrency, diff --git a/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp b/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp index 9318f5fc3cc44..68b09f5598524 100644 --- a/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp +++ b/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -120,6 +121,39 @@ class WorkerFaultBackend : public InMemoryBackend std::set head_keys; }; +class RedeleteDiagnosticsBackend : public InMemoryBackend +{ +public: + std::set blob_keys; + std::function read_blob_hook; + std::function &)> delete_chunk_hook; + + std::optional head(const String & key, TransportAccess & access) override + { + if (!read_blob_hook || !blob_keys.contains(key)) + return InMemoryBackend::head(key, access); + + SCOPE_EXIT({ LOG_WARNING(logger, "Test blob read finished"); }); + read_blob_hook(key); + return InMemoryBackend::head(key, access); + } + + std::vector removeManyByConditional( + const std::vector & removals, TransportAccess & access) override + { + if (!delete_chunk_hook) + return InMemoryBackend::removeManyByConditional(removals, access); + + LOG_WARNING(logger, "Test delete chunk started"); + SCOPE_EXIT({ LOG_WARNING(logger, "Test delete chunk finished"); }); + delete_chunk_hook(removals); + return InMemoryBackend::removeManyByConditional(removals, access); + } + +private: + LoggerPtr logger = getLogger("CasGc"); +}; + class BlobRemoveWitnessBackend : public InMemoryBackend { public: @@ -848,6 +882,129 @@ TEST(CASGCRedeleteConcurrency, TwoWorkerFailuresAreEachCountedAndTheRoundThrowsO EXPECT_TRUE(allBlobsAbsent(*backend, *store)); } +TEST(CASGCRedeleteConcurrency, FailuresAreReportedAfterTheirWaveAndRethrowLowestBlobIndex) +{ + for (const uint64_t concurrency : {1, 4}) + { + SCOPED_TRACE(concurrency); + auto backend = std::make_shared(); + backend->pinConditionalBatchDeleteSupport(true); + auto store = openPoolWithIoConcurrency(backend, concurrency, 1, 0, {}, std::nullopt, 2); + Gc gc(store, kGc); + publishThenDrop(*backend, store, gc); + ASSERT_NO_FATAL_FAILURE(driveUntilAllGraduated(*backend, gc, store, kFaultedBlob)); + const auto preview = previewPendingDeletes(gc); + ASSERT_EQ(preview.size(), kBlobs); + + /// Identical messages require the GC diagnostic to identify the operation itself. + const auto read_error = std::make_exception_ptr(std::runtime_error("injected job fault")); + const auto delete_error = std::make_exception_ptr(std::runtime_error("injected job fault")); + backend->blob_keys = allBlobKeys(*store); + backend->read_blob_hook = [&](const String & key) + { + if (key == preview.back().key) + std::rethrow_exception(read_error); + }; + backend->delete_chunk_hook = [&](const std::vector & removals) + { + if (removals.front().key == preview.front().key) + { + EXPECT_EQ(removals.size(), 2u); + std::rethrow_exception(delete_error); + } + }; + + ScopedLogCapture capture("CasGc", "warning"); + auto rows = std::make_shared(); + gc.setPhaseSink(recordPendingDeletes(rows)); + const auto failed_before = failedCounter(); + RoundReport progress; + std::exception_ptr error; + try + { + gc.runRegularRound({}, /*allow_steal*/ true, UniversePolicy::Authoritative, &progress); + } + catch (...) + { + error = std::current_exception(); + } + backend->read_blob_hook = {}; + backend->delete_chunk_hook = {}; + + const String log = capture.captured(); + const size_t read_warning = log.find("CAS gc: read blob job failed"); + const size_t delete_warning = log.find("CAS gc: delete chunk job failed"); + ASSERT_NE(read_warning, String::npos); + ASSERT_NE(delete_warning, String::npos); + EXPECT_LT(log.rfind("Test blob read finished"), read_warning); + EXPECT_LT(read_warning, log.find("Test delete chunk started")); + EXPECT_LT(log.rfind("Test delete chunk finished"), delete_warning); + EXPECT_EQ(capture.count("CAS gc: read blob job failed"), 1u); + EXPECT_EQ(capture.count("CAS gc: delete chunk job failed"), 2u); + EXPECT_EQ(error, delete_error) << "the lower blob index wins even though its wave is logged later"; + EXPECT_EQ(progress.redeleted, 3u); + EXPECT_EQ(progress.redelete_failed, 3u); + EXPECT_EQ(failedCounter() - failed_before, 3u); + EXPECT_EQ(rows->total("jobs_failed"), 3u); + EXPECT_EQ(rows->total("delete_batches"), 3u); + EXPECT_EQ(rows->total("jobs_scheduled"), concurrency == 1 ? 0u : kBlobs); + EXPECT_EQ(previewPendingDeletes(gc).size(), kBlobs); + for (size_t i = 0; i < preview.size(); ++i) + { + OperationForTest op(*backend); + EXPECT_EQ((*op).head(preview[i].key, Retry::once()).has_value(), i < 2 || i == 5); + } + store->renewWatermarkOnce(); + EXPECT_EQ(runRegularRoundReclaiming(gc).redeleted, kBlobs); + EXPECT_TRUE(allBlobsAbsent(*backend, *store)); + } +} + +TEST(CASGCRedeleteConcurrency, EnqueueFailuresIdentifyWaveAndUnscheduledBlobCount) +{ + struct Case + { + bool delete_wave; + size_t refused_job; + size_t deleted; + size_t unscheduled; + }; + for (const Case test : {Case{false, 0, 0, 6}, Case{false, 3, 3, 3}, Case{true, 0, 0, 6}, Case{true, 1, 4, 2}}) + { + SCOPED_TRACE(test.delete_wave); + SCOPED_TRACE(test.refused_job); + auto backend = std::make_shared(); + backend->pinConditionalBatchDeleteSupport(true); + const size_t refused_index = (test.delete_wave ? kBlobs : 0) + test.refused_job; + auto store = openPoolWithIoConcurrency(backend, 4, 1, 0, {}, refused_index, 4); + Gc gc(store, kGc); + publishThenDrop(*backend, store, gc); + ASSERT_NO_FATAL_FAILURE(driveUntilAllGraduated(*backend, gc, store, kFaultedBlob)); + + ScopedLogCapture capture("CasGc", "warning"); + auto rows = std::make_shared(); + gc.setPhaseSink(recordPendingDeletes(rows)); + const auto failed_before = failedCounter(); + RoundReport progress; + expectThrowsCode(DB::ErrorCodes::CANNOT_SCHEDULE_TASK, [&] + { + gc.runRegularRound({}, /*allow_steal*/ true, UniversePolicy::Authoritative, &progress); + }); + EXPECT_EQ(capture.count(test.delete_wave + ? "CAS gc: scheduling delete chunk job failed" : "CAS gc: scheduling read blob job failed"), 1u); + EXPECT_NE(capture.captured().find(std::to_string(test.unscheduled) + (test.delete_wave + ? " blob deletions were not submitted" : " blob reads were not submitted")), String::npos); + EXPECT_EQ(progress.redeleted, test.deleted); + EXPECT_EQ(progress.redelete_failed, 1u); + EXPECT_EQ(failedCounter() - failed_before, 1u); + EXPECT_EQ(rows->total("jobs_failed"), 1u); + EXPECT_EQ(previewPendingDeletes(gc).size(), kBlobs); + store->renewWatermarkOnce(); + EXPECT_EQ(runRegularRoundReclaiming(gc).redeleted, kBlobs); + EXPECT_TRUE(allBlobsAbsent(*backend, *store)); + } +} + TEST(CASGCRedeleteConcurrency, WorkerHeadFaultIsCountedAndRetriedNextRound) { auto backend = std::make_shared(); From de0efdfc07844898b402839e770a77a08bb2b654 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Wed, 18 Nov 2026 15:57:33 +0100 Subject: [PATCH 20/30] Separate CAS GC read and delete job metrics The pending-deletes `jobs_failed` aggregate mixed failed blobs with enqueue refusals, so a failed multi-blob chunk looked like multiple job failures. Count failed read jobs, failed delete chunk jobs, and enqueue refusals for each wave separately. Include inline execution failures; keep legacy report and profile-event accounting unchanged. Use `read_blob_jobs_scheduled` and `delete_chunk_jobs_scheduled` for successful pool submissions, and `delete_chunks` for logical chunks including inline execution. Update tests, metric consumers, and docs. Co-authored-by: Codex (GPT-6) Signed-off-by: Konstantin Morozov --- .../cas/architecture/garbage-collection.md | 26 +++-- .../en/operations/system-tables/cas_gc_log.md | 25 +++-- .../ContentAddressed/Gc/CasGc.cpp | 37 +++++-- .../ContentAddressed/Gc/CasGc.h | 7 +- .../gtest_cas_gc_redelete_concurrency.cpp | 99 ++++++++++++++----- .../test.py | 23 ++++- 6 files changed, 168 insertions(+), 49 deletions(-) diff --git a/docs/en/antalya/cas/architecture/garbage-collection.md b/docs/en/antalya/cas/architecture/garbage-collection.md index 99bc77d63bf4a..360df9fd2d0d5 100644 --- a/docs/en/antalya/cas/architecture/garbage-collection.md +++ b/docs/en/antalya/cas/architecture/garbage-collection.md @@ -431,9 +431,12 @@ forensic outcome logs. - **Fails the round if:** a backend delete marker appears in a response — object versioning on a mis-provisioned pool (`LOGICAL_ERROR`) - **Observability:** phase row `pending_deletes`; metrics `deleted`, `absent`, `redeleted`, - `graduated`, `replaced`, `spared`, `outcome_logs_written`, `outcome_rows_dropped`, `delete_batches`. `delete_batches` - counts logical conditional-delete chunks submitted by GC. `DiskS3DeleteObjects` counts physical S3 - delete attempts, including request retries + `graduated`, `replaced`, `spared`, `outcome_logs_written`, `outcome_rows_dropped`, `delete_chunks`. `delete_chunks` + counts logical conditional-delete chunks, including sequential execution. `read_blob_jobs_scheduled` + and `delete_chunk_jobs_scheduled` count successfully enqueued read jobs and delete chunk jobs, + respectively. Jobs that later fail still count; enqueue failures do not. Both counters are zero + on the sequential path, including a wave with only one job. `DiskS3DeleteObjects` counts physical + S3 delete attempts, including request retries GC reports read-job failures after all submitted blob-state reads finish, before starting the conditional-delete wave. It reports delete-job failures after all submitted delete chunks finish. @@ -444,9 +447,20 @@ all objects in the chunk remain present. Read failures do not prevent deletion of successfully read siblings. After both waves, GC records known outcomes and schedules eligible `.meta` cleanup, then rethrows the original job or enqueue exception with the lowest input blob index. An unsuccessful round leaves the previously published -pending entries available to the next round. The existing `jobs_failed` metric and -`CASGCRetiredRedeleteFailed` event count each failed blob from an executed job, but count an enqueue -failure once per wave, regardless of the number of unsubmitted blobs reported in its warning. +pending entries available to the next round. + +The `read_blob_jobs_failed` and `delete_chunk_jobs_failed` metrics count jobs that finish with an +exception, including inline execution. A failed delete chunk contributes one job failure, regardless +of its number of blobs. The `read_blob_job_schedule_failures` and `delete_chunk_job_schedule_failures` +metrics each count enqueue refusals, not the number of unsubmitted jobs or blobs. Retries do not +create extra logical jobs; `Gone` and `Mismatch` are normal outcomes, not job failures. These four +metrics replace the former `jobs_failed` aggregate and are emitted even when the phase throws. +Because inline jobs are included only in the execution-failure counters, the scheduling counters +cannot be used as their denominators for an error rate. + +The `RoundReport::redelete_failed` field and `CASGCRetiredRedeleteFailed` event retain their existing +units: each failed blob from an executed job counts once, while an enqueue failure counts once per +wave regardless of the number of unsubmitted blobs reported in its warning. The project RustFS integration backend supports exact-token singleton deletes but does not implement AWS per-object `ETag` conditions in `DeleteObjects`. Its mount probe therefore keeps one-key chunks; diff --git a/docs/en/operations/system-tables/cas_gc_log.md b/docs/en/operations/system-tables/cas_gc_log.md index d08fcb3aaa9d4..dad4001a4672a 100644 --- a/docs/en/operations/system-tables/cas_gc_log.md +++ b/docs/en/operations/system-tables/cas_gc_log.md @@ -81,7 +81,7 @@ The phases, in execution order: | `fold_ref_intake` | Read and fold every new ref log and the manifest bodies its edges name. | one `GET` per new log, one `GET` per manifest edge | | `fold_reduce` | The per-shard in-degree merge: condemn, spare, graduate. | prior-run streaming `GET`s, one `HEAD` per zero-transition candidate, run `PUT`s | | `fold_seal_write` | Publish the new fold seal. | one `PUT` | -| `pending_deletes` | The single content-delete site: exact-token deletes of previously published `delete_pending` entries, plus the outcome logs. `phase_metrics` carries `jobs_scheduled` (entries submitted to the parallel observation wave; `0` for no candidates or sequential execution), `jobs_failed`, and `delete_batches` (logical conditional-delete chunks). Unsupported or unprobed storage uses one-key chunks. | one `HEAD` per entry, conditional delete chunks, one outcome-log `PUT` per shard with rows | +| `pending_deletes` | The single content-delete site: exact-token deletes of previously published `delete_pending` entries, plus the outcome logs. `phase_metrics` carries `read_blob_jobs_scheduled` and `delete_chunk_jobs_scheduled` (read jobs and delete chunk jobs successfully enqueued in the parallel pool), separate execution and enqueue failure counters described below, and `delete_chunks` (logical conditional-delete chunks, including sequential execution). Both scheduling counters are `0` on the sequential path, including waves with at most one job. Unsupported or unprobed storage uses one-key chunks. | one `HEAD` per entry, conditional delete chunks, one outcome-log `PUT` per shard with rows | | `meta_pool_wait` | Drain the round's per-hash freshness-meta writes. | none on this thread — see the caveat below | | `round_commit` | The generation-retention prune and the round's single `gc/state` compare-and-swap. | prune `LIST`s and deletes, one compare-and-swap | | `handoff_reclaim` | Wholesale-reclaim generations a moved run ref stranded below the retention cursor. | prefix `LIST`s and deletes | @@ -124,6 +124,12 @@ WHERE round_id = '...' AND event_type = 'Phase' ORDER BY event_time_microseconds; ``` +For `pending_deletes`, the former `jobs_scheduled` key is now `read_blob_jobs_scheduled`, and +`delete_batches` is now `delete_chunks`. The former `jobs_failed` aggregate is replaced by separate +read/delete execution and enqueue failure counters below. Queries over older rows must use the +keys written by that server version. Other phases, including +`meta_pool_wait`, retain their existing keys. + Two caveats when reading these rows: - Work scheduled onto the GC meta pool runs on other threads, so the `meta_pool_wait` row's @@ -132,11 +138,18 @@ Two caveats when reading these rows: - Requests issued by GC I/O pool workers — the `pending_deletes` fan-out and the fold read-ahead of `fold_ref_intake` and `fold_reduce` — also run on other threads and are missing from those phase rows' `ProfileEvents`. They still count in `system.events`. On `pending_deletes`, read - `phase_metrics` `jobs_scheduled` / `jobs_failed` next to `redeleted`. - `jobs_failed` counts failed blobs from executed read/delete jobs, plus one per enqueue failure. - A failed chunk with multiple blobs therefore contributes multiple failures, while an unsubmitted - tail contributes one enqueue failure. GC warning messages distinguish these cases: read errors - are reported after the read wave completes, and delete errors after the delete wave completes. + `phase_metrics` `read_blob_jobs_scheduled` / `delete_chunk_jobs_scheduled` next to `redeleted`. + The two scheduling counters include enqueued jobs that fail during execution; they exclude jobs + whose enqueue failed and jobs executed inline. `read_blob_jobs_failed` and `delete_chunk_jobs_failed` + count jobs that finish with an exception, including inline jobs. A failed multi-blob chunk counts + once. These execution-failure counters can therefore be nonzero when the scheduling counters are + zero; the latter are not error-rate denominators. `read_blob_job_schedule_failures` and + `delete_chunk_job_schedule_failures` count enqueue refusals once each, regardless of the number of + unsubmitted jobs or blobs. Retries do not multiply logical job counts. All four failure counters + are emitted, including zeros and phase exits with an exception. Read errors are reported after + the read wave completes, and delete errors after the delete wave completes. The `Finish` row's + `entries_redelete_failed` and `CASGCRetiredRedeleteFailed` retain the legacy units: failed blobs + from executed jobs plus one per enqueue refusal. - Phase durations do not sum to the round's `duration_ms`. The round also performs untimed bookkeeping between phases, and the `Finish` row's `duration_ms` remains the authority on total round time. diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp index a3787767a5670..98dd06e6ac244 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp @@ -807,7 +807,7 @@ void Gc::redeleteBlobs( [&](const size_t index, CasOperation & job_op) { readRetiredBlobState(entries[index], layout, job_op, redelete_blob_states[index]); }); if (read_blob_jobs.on_pool) - ctx.jobs_scheduled += read_blob_jobs.scheduled; + ctx.read_blob_jobs_scheduled += read_blob_jobs.scheduled; std::optional first_error = recordReadBlobJobFailures(ctx, entries, layout, redelete_blob_states, read_blob_jobs); @@ -829,6 +829,8 @@ void Gc::redeleteBlobs( redelete_blob_states[delete_chunk.blob_state_indices[pos]].removal_result = removal_results[pos]; }); + if (delete_chunk_jobs.on_pool) + ctx.delete_chunk_jobs_scheduled += delete_chunk_jobs.scheduled; ctx.delete_chunks += delete_chunk_jobs.scheduled; const std::optional delete_error @@ -858,12 +860,14 @@ std::optional Gc::recordReadBlobJobFailures( redelete_blob_states[i].error = error; if (!error) continue; + ++ctx.read_blob_jobs_failed; recordRedeleteFailure(ctx, entries[i], layout, RedeleteFailureKind::ReadBlobJob, error); if (!first_error || i < first_error->index) first_error = IndexedException{i, error}; } if (read_blob_jobs.job_schedule_error) { + ++ctx.read_blob_job_schedule_failures; const IndexedException & error = *read_blob_jobs.job_schedule_error; chassert(error.index < entries.size()); recordRedeleteFailure(ctx, entries[error.index], layout, RedeleteFailureKind::ScheduleReadBlobJob, @@ -888,6 +892,7 @@ std::optional Gc::recordDeleteChunkJobFailures( const std::exception_ptr & error = delete_chunk_jobs.scheduled_job_errors[chunk_index]; if (!error) continue; + ++ctx.delete_chunk_jobs_failed; for (const size_t index : delete_chunks[chunk_index].blob_state_indices) { redelete_blob_states[index].error = error; @@ -898,6 +903,7 @@ std::optional Gc::recordDeleteChunkJobFailures( } if (delete_chunk_jobs.job_schedule_error) { + ++ctx.delete_chunk_job_schedule_failures; const size_t first_unscheduled_chunk_index = delete_chunk_jobs.job_schedule_error->index; chassert(first_unscheduled_chunk_index < delete_chunks.size()); size_t unscheduled_delete_blob_count = 0; @@ -1225,14 +1231,22 @@ RoundReport Gc::runRegularRound(std::function on_lease_acquired, bool al pending_deletes_timer.emplace(phase_sink, "pending_deletes"); const uint64_t redeleted_before = report.redeleted; const uint64_t graduated_before = report.graduated; - const uint64_t redelete_failed_before = report.redelete_failed; - uint64_t redelete_jobs_scheduled = 0; - uint64_t redelete_chunks = 0; + uint64_t read_blob_jobs_scheduled = 0; + uint64_t delete_chunk_jobs_scheduled = 0; + uint64_t read_blob_jobs_failed = 0; + uint64_t delete_chunk_jobs_failed = 0; + uint64_t read_blob_job_schedule_failures = 0; + uint64_t delete_chunk_job_schedule_failures = 0; + uint64_t delete_chunks = 0; const auto record_redelete_job_metrics = [&] { - pending_deletes_timer->metric("jobs_scheduled", redelete_jobs_scheduled); - pending_deletes_timer->metric("jobs_failed", report.redelete_failed - redelete_failed_before); - pending_deletes_timer->metric("delete_batches", redelete_chunks); + pending_deletes_timer->metric("read_blob_jobs_scheduled", read_blob_jobs_scheduled); + pending_deletes_timer->metric("delete_chunk_jobs_scheduled", delete_chunk_jobs_scheduled); + pending_deletes_timer->metric("read_blob_jobs_failed", read_blob_jobs_failed); + pending_deletes_timer->metric("delete_chunk_jobs_failed", delete_chunk_jobs_failed); + pending_deletes_timer->metric("read_blob_job_schedule_failures", read_blob_job_schedule_failures); + pending_deletes_timer->metric("delete_chunk_job_schedule_failures", delete_chunk_job_schedule_failures); + pending_deletes_timer->metric("delete_chunks", delete_chunks); }; SCOPE_EXIT_SAFE({ if (pending_deletes_timer) @@ -1268,8 +1282,13 @@ RoundReport Gc::runRegularRound(std::function on_lease_acquired, bool al .report = report, .outcomes = outcomes, .outcome_tallies = outcome_tallies, - .jobs_scheduled = redelete_jobs_scheduled, - .delete_chunks = redelete_chunks}; + .read_blob_jobs_scheduled = read_blob_jobs_scheduled, + .delete_chunk_jobs_scheduled = delete_chunk_jobs_scheduled, + .read_blob_jobs_failed = read_blob_jobs_failed, + .delete_chunk_jobs_failed = delete_chunk_jobs_failed, + .read_blob_job_schedule_failures = read_blob_job_schedule_failures, + .delete_chunk_job_schedule_failures = delete_chunk_job_schedule_failures, + .delete_chunks = delete_chunks}; redeleteBlobs(redelete_ctx, redelete_now, *io_pool, store->poolConfig().gc_io_concurrency, layout, op); diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h index 80178b9fed077..0181d0704b7e0 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h @@ -827,7 +827,12 @@ class Gc RoundReport & report; std::map & outcomes; std::map & outcome_tallies; - uint64_t & jobs_scheduled; + uint64_t & read_blob_jobs_scheduled; + uint64_t & delete_chunk_jobs_scheduled; + uint64_t & read_blob_jobs_failed; + uint64_t & delete_chunk_jobs_failed; + uint64_t & read_blob_job_schedule_failures; + uint64_t & delete_chunk_job_schedule_failures; uint64_t & delete_chunks; }; diff --git a/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp b/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp index 68b09f5598524..d8b4ba40546b0 100644 --- a/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp +++ b/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp @@ -414,8 +414,12 @@ TEST(CASGCRedeleteConcurrency, ParallelRedeleteReclaimsEveryBlob) EXPECT_EQ(redeleted, kBlobs); EXPECT_EQ(deleted, kBlobs); EXPECT_EQ(redelete_failed, 0u); - EXPECT_EQ(rows->total("jobs_scheduled"), kBlobs); - EXPECT_EQ(rows->total("jobs_failed"), 0u); + EXPECT_EQ(rows->total("read_blob_jobs_scheduled"), kBlobs); + EXPECT_EQ(rows->total("delete_chunk_jobs_scheduled"), kBlobs); + EXPECT_EQ(rows->total("read_blob_jobs_failed"), 0u); + EXPECT_EQ(rows->total("delete_chunk_jobs_failed"), 0u); + EXPECT_EQ(rows->total("read_blob_job_schedule_failures"), 0u); + EXPECT_EQ(rows->total("delete_chunk_job_schedule_failures"), 0u); } TEST(CASGCRedeleteConcurrency, ConditionalDeleteBatchesAreRecorded) @@ -434,7 +438,17 @@ TEST(CASGCRedeleteConcurrency, ConditionalDeleteBatchesAreRecorded) EXPECT_EQ(rep.deleted, kBlobs); ASSERT_EQ(rows->rows.size(), 1u); - EXPECT_EQ(rows->total("delete_batches"), 3u); + EXPECT_EQ(rows->total("read_blob_jobs_scheduled"), kBlobs); + EXPECT_EQ(rows->total("delete_chunk_jobs_scheduled"), 3u); + EXPECT_EQ(rows->total("delete_chunks"), 3u); + ASSERT_TRUE(rows->lastRowHas("read_blob_jobs_failed")); + EXPECT_EQ(rows->total("read_blob_jobs_failed"), 0u); + ASSERT_TRUE(rows->lastRowHas("delete_chunk_jobs_failed")); + EXPECT_EQ(rows->total("delete_chunk_jobs_failed"), 0u); + ASSERT_TRUE(rows->lastRowHas("read_blob_job_schedule_failures")); + EXPECT_EQ(rows->total("read_blob_job_schedule_failures"), 0u); + ASSERT_TRUE(rows->lastRowHas("delete_chunk_job_schedule_failures")); + EXPECT_EQ(rows->total("delete_chunk_job_schedule_failures"), 0u); } TEST(CASGCRedeleteConcurrency, DeleteWaveEnqueueRefusalWaitsForSubmittedChunks) @@ -498,8 +512,19 @@ TEST(CASGCRedeleteConcurrency, ConcurrencyOneDeletesSequentially) EXPECT_TRUE(allBlobsAbsent(*backend, *store)); EXPECT_EQ(backend->blobRemoves(), kBlobs); EXPECT_EQ(backend->peakInFlight(), 1u); - ASSERT_TRUE(rows->lastRowHas("jobs_scheduled")); - EXPECT_EQ(rows->total("jobs_scheduled"), 0u) << "concurrency 1 must not use the pool"; + ASSERT_TRUE(rows->lastRowHas("read_blob_jobs_scheduled")); + ASSERT_TRUE(rows->lastRowHas("delete_chunk_jobs_scheduled")); + EXPECT_EQ(rows->total("read_blob_jobs_scheduled"), 0u) << "concurrency 1 must not use the pool"; + EXPECT_EQ(rows->total("delete_chunk_jobs_scheduled"), 0u); + EXPECT_EQ(rows->total("delete_chunks"), kBlobs); + ASSERT_TRUE(rows->lastRowHas("read_blob_jobs_failed")); + EXPECT_EQ(rows->total("read_blob_jobs_failed"), 0u); + ASSERT_TRUE(rows->lastRowHas("delete_chunk_jobs_failed")); + EXPECT_EQ(rows->total("delete_chunk_jobs_failed"), 0u); + ASSERT_TRUE(rows->lastRowHas("read_blob_job_schedule_failures")); + EXPECT_EQ(rows->total("read_blob_job_schedule_failures"), 0u); + ASSERT_TRUE(rows->lastRowHas("delete_chunk_job_schedule_failures")); + EXPECT_EQ(rows->total("delete_chunk_job_schedule_failures"), 0u); } TEST(CASGCRedeleteConcurrency, WorkerRemoveFaultKeepsSiblingOutcomesAndPoolAlive) @@ -870,9 +895,12 @@ TEST(CASGCRedeleteConcurrency, TwoWorkerFailuresAreEachCountedAndTheRoundThrowsO EXPECT_EQ(progress.redeleted, kBlobs - 2); EXPECT_EQ(progress.redelete_failed, 2u); ASSERT_EQ(rows->rows.size(), 1u); - ASSERT_TRUE(rows->lastRowHas("jobs_failed")) << "the phase metrics must survive the round's exception"; - EXPECT_EQ(rows->total("jobs_scheduled"), kBlobs); - EXPECT_EQ(rows->total("jobs_failed"), 2u); + ASSERT_TRUE(rows->lastRowHas("delete_chunk_jobs_failed")) << "the phase metrics must survive the round's exception"; + EXPECT_EQ(rows->total("read_blob_jobs_scheduled"), kBlobs); + EXPECT_EQ(rows->total("read_blob_jobs_failed"), 0u); + EXPECT_EQ(rows->total("delete_chunk_jobs_failed"), 2u); + EXPECT_EQ(rows->total("read_blob_job_schedule_failures"), 0u); + EXPECT_EQ(rows->total("delete_chunk_job_schedule_failures"), 0u); expectBodiesPresentOnlyFor(*backend, *store, {kFaultedBlob, kReplacedBlob}); const RoundReport next = runRegularRoundReclaiming(gc); @@ -945,9 +973,13 @@ TEST(CASGCRedeleteConcurrency, FailuresAreReportedAfterTheirWaveAndRethrowLowest EXPECT_EQ(progress.redeleted, 3u); EXPECT_EQ(progress.redelete_failed, 3u); EXPECT_EQ(failedCounter() - failed_before, 3u); - EXPECT_EQ(rows->total("jobs_failed"), 3u); - EXPECT_EQ(rows->total("delete_batches"), 3u); - EXPECT_EQ(rows->total("jobs_scheduled"), concurrency == 1 ? 0u : kBlobs); + EXPECT_EQ(rows->total("read_blob_jobs_failed"), 1u); + EXPECT_EQ(rows->total("delete_chunk_jobs_failed"), 1u); + EXPECT_EQ(rows->total("read_blob_job_schedule_failures"), 0u); + EXPECT_EQ(rows->total("delete_chunk_job_schedule_failures"), 0u); + EXPECT_EQ(rows->total("delete_chunks"), 3u); + EXPECT_EQ(rows->total("read_blob_jobs_scheduled"), concurrency == 1 ? 0u : kBlobs); + EXPECT_EQ(rows->total("delete_chunk_jobs_scheduled"), concurrency == 1 ? 0u : 3u); EXPECT_EQ(previewPendingDeletes(gc).size(), kBlobs); for (size_t i = 0; i < preview.size(); ++i) { @@ -968,8 +1000,10 @@ TEST(CASGCRedeleteConcurrency, EnqueueFailuresIdentifyWaveAndUnscheduledBlobCoun size_t refused_job; size_t deleted; size_t unscheduled; + size_t read_blob_jobs_scheduled; + size_t delete_chunk_jobs_scheduled; }; - for (const Case test : {Case{false, 0, 0, 6}, Case{false, 3, 3, 3}, Case{true, 0, 0, 6}, Case{true, 1, 4, 2}}) + for (const Case test : {Case{false, 0, 0, 6, 0, 0}, Case{false, 3, 3, 3, 3, 0}, Case{true, 0, 0, 6, 6, 0}, Case{true, 1, 4, 2, 6, 1}}) { SCOPED_TRACE(test.delete_wave); SCOPED_TRACE(test.refused_job); @@ -997,7 +1031,14 @@ TEST(CASGCRedeleteConcurrency, EnqueueFailuresIdentifyWaveAndUnscheduledBlobCoun EXPECT_EQ(progress.redeleted, test.deleted); EXPECT_EQ(progress.redelete_failed, 1u); EXPECT_EQ(failedCounter() - failed_before, 1u); - EXPECT_EQ(rows->total("jobs_failed"), 1u); + ASSERT_TRUE(rows->lastRowHas("read_blob_jobs_scheduled")); + ASSERT_TRUE(rows->lastRowHas("delete_chunk_jobs_scheduled")); + EXPECT_EQ(rows->total("read_blob_jobs_scheduled"), test.read_blob_jobs_scheduled); + EXPECT_EQ(rows->total("delete_chunk_jobs_scheduled"), test.delete_chunk_jobs_scheduled); + EXPECT_EQ(rows->total("read_blob_jobs_failed"), 0u); + EXPECT_EQ(rows->total("delete_chunk_jobs_failed"), 0u); + EXPECT_EQ(rows->total("read_blob_job_schedule_failures"), test.delete_wave ? 0u : 1u); + EXPECT_EQ(rows->total("delete_chunk_job_schedule_failures"), test.delete_wave ? 1u : 0u); EXPECT_EQ(previewPendingDeletes(gc).size(), kBlobs); store->renewWatermarkOnce(); EXPECT_EQ(runRegularRoundReclaiming(gc).redeleted, kBlobs); @@ -1026,8 +1067,11 @@ TEST(CASGCRedeleteConcurrency, WorkerHeadFaultIsCountedAndRetriedNextRound) EXPECT_EQ(progress.redeleted, kBlobs - 1); EXPECT_EQ(progress.redelete_failed, 1u); ASSERT_EQ(rows->rows.size(), 1u); - EXPECT_EQ(rows->total("jobs_scheduled"), kBlobs); - EXPECT_EQ(rows->total("jobs_failed"), 1u); + EXPECT_EQ(rows->total("read_blob_jobs_scheduled"), kBlobs); + EXPECT_EQ(rows->total("read_blob_jobs_failed"), 1u); + EXPECT_EQ(rows->total("delete_chunk_jobs_failed"), 0u); + EXPECT_EQ(rows->total("read_blob_job_schedule_failures"), 0u); + EXPECT_EQ(rows->total("delete_chunk_job_schedule_failures"), 0u); expectBodiesPresentOnlyFor(*backend, *store, {kFaultedBlob}); const RoundReport next = runRegularRoundReclaiming(gc); @@ -1082,9 +1126,12 @@ TEST(CASGCRedeleteConcurrency, SchedulingFailureAttemptsNothingAndTheNextRoundRe EXPECT_EQ(progress.redelete_failed, 1u); EXPECT_EQ(failedCounter() - failed_before, 1u); ASSERT_EQ(rows->rows.size(), 1u); - ASSERT_TRUE(rows->lastRowHas("jobs_scheduled")); - EXPECT_EQ(rows->total("jobs_scheduled"), 0u); - EXPECT_EQ(rows->total("jobs_failed"), 1u); + ASSERT_TRUE(rows->lastRowHas("read_blob_jobs_scheduled")); + EXPECT_EQ(rows->total("read_blob_jobs_scheduled"), 0u); + EXPECT_EQ(rows->total("read_blob_jobs_failed"), 0u); + EXPECT_EQ(rows->total("delete_chunk_jobs_failed"), 0u); + EXPECT_EQ(rows->total("read_blob_job_schedule_failures"), 1u); + EXPECT_EQ(rows->total("delete_chunk_job_schedule_failures"), 0u); expectBodiesPresentOnlyFor(*backend, *store, {1, 2, 3, 4, 5, 6}); const RoundReport next = runRegularRoundReclaiming(gc); @@ -1122,8 +1169,11 @@ TEST(CASGCRedeleteConcurrency, EnqueueRefusalYieldsToLowerIndexWorkerFailure) EXPECT_EQ(progress.redelete_failed, 2u); EXPECT_EQ(failedCounter() - failed_before, 2u); - EXPECT_EQ(rows->total("jobs_scheduled"), 3u); - EXPECT_EQ(rows->total("jobs_failed"), 2u); + EXPECT_EQ(rows->total("read_blob_jobs_scheduled"), 3u); + EXPECT_EQ(rows->total("read_blob_jobs_failed"), 0u); + EXPECT_EQ(rows->total("delete_chunk_jobs_failed"), 1u); + EXPECT_EQ(rows->total("read_blob_job_schedule_failures"), 1u); + EXPECT_EQ(rows->total("delete_chunk_job_schedule_failures"), 0u); EXPECT_NE(message.find(first_preview_key), String::npos); EXPECT_EQ(message.find("Injected CAS GC re-delete enqueue refusal"), String::npos); EXPECT_TRUE(backend->allFired()); @@ -1175,8 +1225,13 @@ TEST(CASGCRedeleteConcurrency, SingletonWorkerFailureIsCountedWithoutScheduling) EXPECT_TRUE(backend->allFired()); EXPECT_EQ(progress.redelete_failed, 1u); EXPECT_EQ(failedCounter() - failed_before, 1u); - EXPECT_EQ(rows->total("jobs_scheduled"), 0u); - EXPECT_EQ(rows->total("jobs_failed"), 1u); + EXPECT_EQ(rows->total("read_blob_jobs_scheduled"), 0u); + ASSERT_TRUE(rows->lastRowHas("delete_chunk_jobs_scheduled")); + EXPECT_EQ(rows->total("delete_chunk_jobs_scheduled"), 0u); + EXPECT_EQ(rows->total("read_blob_jobs_failed"), 0u); + EXPECT_EQ(rows->total("delete_chunk_jobs_failed"), 1u); + EXPECT_EQ(rows->total("read_blob_job_schedule_failures"), 0u); + EXPECT_EQ(rows->total("delete_chunk_job_schedule_failures"), 0u); } TEST(CASGCRedeleteConcurrency, ShardedPoolDeletesEveryShard) diff --git a/tests/integration/test_cas_gc_conditional_batch_delete/test.py b/tests/integration/test_cas_gc_conditional_batch_delete/test.py index 32d9a72dd677f..5c866f339f32d 100644 --- a/tests/integration/test_cas_gc_conditional_batch_delete/test.py +++ b/tests/integration/test_cas_gc_conditional_batch_delete/test.py @@ -50,8 +50,11 @@ def gc_pending_delete_rows(node, after_round): node.query("SYSTEM FLUSH LOGS") rows = ( node.query( - "SELECT p.phase_metrics['redeleted'], p.phase_metrics['delete_batches'], " - "p.phase_metrics['jobs_failed'] " + "SELECT p.phase_metrics['redeleted'], p.phase_metrics['delete_chunks'], " + "p.phase_metrics['read_blob_jobs_failed'], " + "p.phase_metrics['delete_chunk_jobs_failed'], " + "p.phase_metrics['read_blob_job_schedule_failures'], " + "p.phase_metrics['delete_chunk_job_schedule_failures'] " "FROM system.cas_gc_log AS p INNER JOIN " "(SELECT round_id FROM system.cas_gc_log " f"WHERE disk_name = '{DISK_NAME}' AND event_type = 'Finish' " @@ -109,7 +112,17 @@ def test_rustfs_pending_blob_deletes_use_singleton_fallback(): assert not remaining, f"GC left blob bodies in RustFS: {sorted(remaining)}" assert rows, "pending_deletes never reported completed blob redeletes" - for redeleted, batches, jobs_failed in rows: - assert jobs_failed == 0 - assert batches == redeleted + for ( + redeleted, + delete_chunks, + read_blob_jobs_failed, + delete_chunk_jobs_failed, + read_blob_job_schedule_failures, + delete_chunk_job_schedule_failures, + ) in rows: + assert read_blob_jobs_failed == 0 + assert delete_chunk_jobs_failed == 0 + assert read_blob_job_schedule_failures == 0 + assert delete_chunk_job_schedule_failures == 0 + assert delete_chunks == redeleted assert gc_redeleted_total(node, first_round) >= len(created_blobs) From d45f61e4dd36919fff4da2c64c926efda606874f Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Wed, 18 Nov 2026 15:57:36 +0100 Subject: [PATCH 21/30] Use death tests for CAS batch-delete logical errors `LOGICAL_ERROR` aborts in debug and sanitizer builds before `EXPECT_THROW` can handle it, terminating the unit-test process. Use `EXPECT_DEATH` for oversized conditional batches and duplicate S3 response keys in those builds; retain exception checks in release. Construct the scripted HTTP server inside the death-test child. CI log: https://altinity-build-artifacts.s3.amazonaws.com/PRs/2534/d05e9bdde03c9e602fe485f8c33e46a63d287854/unit_tests_asan_ubsan/job.log Related: https://github.com/Altinity/ClickHouse/pull/2534 Co-authored-by: Codex (GPT-6) Signed-off-by: Konstantin Morozov --- .../tests/gtest_cas_bulk_delete_engine.cpp | 9 ++++ .../gtest_cas_s3_bulk_delete_fallback.cpp | 43 ++++++++++++------- 2 files changed, 36 insertions(+), 16 deletions(-) diff --git a/src/Disks/tests/gtest_cas_bulk_delete_engine.cpp b/src/Disks/tests/gtest_cas_bulk_delete_engine.cpp index 48a16b7a8df87..7605c98d02ee3 100644 --- a/src/Disks/tests/gtest_cas_bulk_delete_engine.cpp +++ b/src/Disks/tests/gtest_cas_bulk_delete_engine.cpp @@ -5,6 +5,7 @@ #include #include #include +#include /// The engine sends one chunk of at most 1000 write-once keys as one request under the ordinary /// attempt loop; a failed attempt reissues the whole chunk, which is sound because a key the failed @@ -125,7 +126,11 @@ TEST(CASBulkDeleteEngineDeathTest, MoreThanOneThousandKeysIsACallerBug) } #endif +#if defined(DEBUG_OR_SANITIZER_BUILD) +TEST(CASConditionalBatchDeleteEngineDeathTest, MoreThanOneThousandKeysIsRejectedEvenIfBackendReportsALargerLimit) +#else TEST(CASConditionalBatchDeleteEngine, MoreThanOneThousandKeysIsRejectedEvenIfBackendReportsALargerLimit) +#endif { auto backend = std::make_shared(); backend->pinConditionalBatchDeleteSupport(true); @@ -139,5 +144,9 @@ TEST(CASConditionalBatchDeleteEngine, MoreThanOneThousandKeysIsRejectedEvenIfBac removals.push_back({key, std::get(op.create(key, "v", Retry::once())).etag}); } +#if defined(DEBUG_OR_SANITIZER_BUILD) + EXPECT_DEATH({ op.removeManyByConditional(removals, Retry::once()); }, "CAS removeManyByConditional: 1001 keys"); +#else EXPECT_THROW(op.removeManyByConditional(removals, Retry::once()), DB::Exception); +#endif } diff --git a/src/Disks/tests/gtest_cas_s3_bulk_delete_fallback.cpp b/src/Disks/tests/gtest_cas_s3_bulk_delete_fallback.cpp index 1750b00b5bb19..ae9de309d73ed 100644 --- a/src/Disks/tests/gtest_cas_s3_bulk_delete_fallback.cpp +++ b/src/Disks/tests/gtest_cas_s3_bulk_delete_fallback.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -433,28 +434,38 @@ TEST(CASS3ConditionalBatchDelete, DoesNotTreatMissingBucketAsMissingObject) } } +#if defined(DEBUG_OR_SANITIZER_BUILD) +TEST(CASS3ConditionalBatchDeleteDeathTest, RejectsDuplicateResponseKey) +#else TEST(CASS3ConditionalBatchDelete, RejectsDuplicateResponseKey) +#endif { - (void)contextForTest(); - - ScriptedS3Server server([](const Poco::Net::HTTPServerRequest &, Poco::Net::HTTPServerResponse & response) + /// Construct the server in the death-test child rather than forking its running threads. + const auto send_duplicate_response = [] { - const std::string body = - "" - "" - "first" - "first" - "secondPreconditionFailedstale" - ""; - sendXml(response, Poco::Net::HTTPResponse::HTTP_OK, body); - }); - auto storage = makeStorageForTest(server.getUrl(), DB::S3Capabilities{}); + (void)contextForTest(); + ScriptedS3Server server([](const Poco::Net::HTTPServerRequest &, Poco::Net::HTTPServerResponse & response) + { + const std::string body = + "" + "" + "first" + "first" + "secondPreconditionFailedstale" + ""; + sendXml(response, Poco::Net::HTTPResponse::HTTP_OK, body); + }); + auto storage = makeStorageForTest(server.getUrl(), DB::S3Capabilities{}); - EXPECT_THROW( storage->removeObjectsIfTokensMatch( {{DB::StoredObject("first"), "etag-first"}, {DB::StoredObject("second"), "etag-second"}}, - DB::ObjectStorageControlRequest{}), - DB::Exception); + DB::ObjectStorageControlRequest{}); + }; +#if defined(DEBUG_OR_SANITIZER_BUILD) + EXPECT_DEATH(send_duplicate_response(), "unexpected or duplicate key first"); +#else + EXPECT_THROW(send_duplicate_response(), DB::Exception); +#endif } TEST(S3BulkDeleteFallback, UnsupportedBatchReplyRecordsCapabilityFalseAndThrowsNotImplemented) From 64813a73b90e431fdf29ee20d1c0da9cb9ea522f Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Wed, 18 Nov 2026 16:30:55 +0100 Subject: [PATCH 22/30] Verify mixed outcomes in the CAS conditional batch probe Exercise two matching tokens, a stale token, and an already removed key in one request. Require `Gone` for the absent key, check that both matching objects are gone before cleanup, and verify that the stale request left the replacement intact. Cover false success and an incorrect outcome for the absent key with regression tests. Share the request implementation through a private `removeManyByConditional` overload with an explicit limit, and avoid copying response keys during S3 validation. Co-authored-by: Codex (GPT-6.2 Sol) Signed-off-by: Konstantin Morozov --- .../ContentAddressed/Backend/CasProbe.cpp | 18 ++++-- .../ContentAddressed/Backend/CasRequests.cpp | 39 +++++++------ .../ContentAddressed/Backend/CasRequests.h | 6 +- .../ObjectStorages/S3/S3ObjectStorage.cpp | 4 +- src/Disks/tests/gtest_cas_probe.cpp | 58 ++++++++++++++++++- 5 files changed, 94 insertions(+), 31 deletions(-) diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasProbe.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasProbe.cpp index 20d08f9fbdef6..cd31bb04c3303 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasProbe.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasProbe.cpp @@ -51,6 +51,7 @@ CapabilityProbeResult runCapabilityProbe(CasOperation & op, const String & probe const String key = probe_prefix + "/token"; const String first_key = probe_prefix + "/batch-first"; const String second_key = probe_prefix + "/batch-second"; + const String third_key = probe_prefix + "/batch-third"; // Best-effort cleanup — runs at function exit regardless of outcome. auto cleanup = [&]() noexcept @@ -61,7 +62,7 @@ CapabilityProbeResult runCapabilityProbe(CasOperation & op, const String & probe // constructible here — the gate below is the only way to reach `remove` at all. try { - for (const String * cleanup_key : {&key, &first_key, &second_key}) + for (const String * cleanup_key : {&key, &first_key, &second_key, &third_key}) { const auto h = op.head(*cleanup_key, Retry::standard()); if (h) @@ -205,14 +206,21 @@ CapabilityProbeResult runCapabilityProbe(CasOperation & op, const String & probe const Etag first_token = std::get(op.create(first_key, "probe-batch-first", Retry::standard())).etag; const Etag second_token = std::get(op.create(second_key, "probe-batch-second-v1", Retry::standard())).etag; const Etag second_replacement = std::get(op.replace(second_key, "probe-batch-second-v2", second_token, Retry::standard())).etag; + const Etag third_token = std::get(op.create(third_key, "probe-batch-third", Retry::standard())).etag; try { - const std::vector removals = op.removeManyByConditionalForProbe( - {{first_key, first_token}, {second_key, second_token}}, Retry::standard()); - if (removals.size() == 2 && removals[0] == Removal::Removed && removals[1] == Removal::Mismatch) + /// Step 7 already deleted `key` using `t2` and verified that it was gone. + const std::vector removals = op.removeManyByConditional( + {{first_key, first_token}, {second_key, second_token}, {third_key, third_token}, {key, t2}}, + Retry::standard(), kBulkDeleteMaxKeys); + if (removals.size() == 4 && removals[0] == Removal::Removed && removals[1] == Removal::Mismatch + && removals[2] == Removal::Removed && removals[3] == Removal::Gone) { + /// Check before cleanup, which would otherwise hide a successful response without deletion. + const auto first = op.head(first_key, Retry::standard()); const auto second = op.read(second_key, Retry::standard()); - result.conditional_batch_delete_supported = second && second->bytes == "probe-batch-second-v2"; + const auto third = op.head(third_key, Retry::standard()); + result.conditional_batch_delete_supported = !first && !third && second && second->bytes == "probe-batch-second-v2"; } } catch (const DB::Exception & e) diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.cpp index 70e0f48a58916..01c7786ccad9b 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.cpp @@ -650,20 +650,32 @@ void CasOperation::removeManyWriteOnce(const std::vector & keys, c }); } -std::vector CasOperation::removeManyByConditional( - const std::vector & removals, const Retry & policy) +namespace { - const size_t limit = std::min(kBulkDeleteMaxKeys, owner.backend->conditionalBatchDeleteKeyLimit()); - return removeManyByConditionalImpl(removals, policy, limit); +Removal toConditionalRemoval(const Backend::RawRemoval raw) +{ + switch (raw) + { + case Backend::RawRemoval::Removed: return Removal::Removed; + case Backend::RawRemoval::Gone: return Removal::Gone; + case Backend::RawRemoval::Mismatch: return Removal::Mismatch; + case Backend::RawRemoval::DeleteMarker: + throw Exception(ErrorCodes::CAS_DELETE_MARKER, + "CAS conditional batch removal archived a noncurrent version instead of reclaiming an object: " + "the bucket has object versioning enabled"); + } + UNREACHABLE(); +} } -std::vector CasOperation::removeManyByConditionalForProbe( +std::vector CasOperation::removeManyByConditional( const std::vector & removals, const Retry & policy) { - return removeManyByConditionalImpl(removals, policy, kBulkDeleteMaxKeys); + const size_t limit = std::min(kBulkDeleteMaxKeys, owner.backend->conditionalBatchDeleteKeyLimit()); + return removeManyByConditional(removals, policy, limit); } -std::vector CasOperation::removeManyByConditionalImpl( +std::vector CasOperation::removeManyByConditional( const std::vector & removals, const Retry & policy, const size_t limit) { if (removals.size() > limit) @@ -698,18 +710,7 @@ std::vector CasOperation::removeManyByConditionalImpl( std::vector results; results.reserve(raw_results.size()); for (const Backend::RawRemoval raw : raw_results) - { - switch (raw) - { - case Backend::RawRemoval::Removed: results.push_back(Removal::Removed); break; - case Backend::RawRemoval::Gone: results.push_back(Removal::Gone); break; - case Backend::RawRemoval::Mismatch: results.push_back(Removal::Mismatch); break; - case Backend::RawRemoval::DeleteMarker: - throw Exception(ErrorCodes::CAS_DELETE_MARKER, - "CAS conditional batch removal archived a noncurrent version instead of reclaiming an object: " - "the bucket has object versioning enabled"); - } - } + results.push_back(toConditionalRemoval(raw)); return results; } diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.h index a93ded0afba05..202282a54c541 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.h @@ -376,10 +376,8 @@ class CasOperation private: friend CapabilityProbeResult runCapabilityProbe(CasOperation & op, const String & probe_prefix); - std::vector removeManyByConditionalForProbe( - const std::vector & removals, const Retry & policy); - std::vector removeManyByConditionalImpl( - const std::vector & removals, const Retry & policy, size_t limit); + std::vector removeManyByConditional( + const std::vector & removals, const Retry & policy, const size_t limit); friend class CasRequests; /// The lane waits on this operation's own admission and reads and writes through its verbs; it diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp index 2084e956f14ed..a517c505ac0d6 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp @@ -718,7 +718,7 @@ std::vector S3ObjectStorage::removeObjectsIfTokensMatch response_keys.reserve(objects.size()); for (const auto & deleted : outcome.GetResult().GetDeleted()) { - const String key = deleted.GetKey(); + const auto & key = deleted.GetKey(); if (!expected_keys.contains(key) || !response_keys.insert(key).second) { log_all(static_cast(Aws::S3::S3Errors::UNKNOWN), "DeleteObjects response has unexpected or duplicate keys"); @@ -727,7 +727,7 @@ std::vector S3ObjectStorage::removeObjectsIfTokensMatch } for (const auto & error : outcome.GetResult().GetErrors()) { - const String key = error.GetKey(); + const auto & key = error.GetKey(); if (!expected_keys.contains(key) || !response_keys.insert(key).second) { log_all(static_cast(Aws::S3::S3Errors::UNKNOWN), "DeleteObjects response has unexpected or duplicate keys"); diff --git a/src/Disks/tests/gtest_cas_probe.cpp b/src/Disks/tests/gtest_cas_probe.cpp index a3c8d4d26e096..b15b716d8b9f5 100644 --- a/src/Disks/tests/gtest_cas_probe.cpp +++ b/src/Disks/tests/gtest_cas_probe.cpp @@ -53,6 +53,43 @@ class IgnoresConditionalBatchTokensBackend : public InMemoryBackend } }; +class NoOpConditionalBatchDeleteBackend : public InMemoryBackend +{ +public: + std::vector removeManyByConditional( + const std::vector & removals, TransportAccess & access) override + { + std::vector results; + results.reserve(removals.size()); + for (const RawConditionalRemove & removal : removals) + { + const auto current = InMemoryBackend::head(removal.key, access); + if (!current) + results.push_back(RawRemoval::Gone); + else if (current->value != removal.expected_token) + results.push_back(RawRemoval::Mismatch); + else + results.push_back(RawRemoval::Removed); + } + return results; + } +}; + +class ReportsAbsentKeyAsRemovedBackend : public InMemoryBackend +{ +public: + std::vector removeManyByConditional( + const std::vector & removals, TransportAccess & access) override + { + auto results = InMemoryBackend::removeManyByConditional(removals, access); + EXPECT_EQ(results, (std::vector{RawRemoval::Removed, RawRemoval::Mismatch, RawRemoval::Removed, RawRemoval::Gone})); + for (auto & result : results) + if (result == RawRemoval::Gone) + result = RawRemoval::Removed; + return results; + } +}; + } TEST(CASProbe, ReportsConditionalBatchCapability) @@ -68,6 +105,25 @@ TEST(CASProbe, ReportsConditionalBatchCapability) EXPECT_FALSE(runCapabilityProbe(ignores_op, "p/.cas_probe").conditional_batch_delete_supported); } +TEST(CASProbe, RejectsConditionalBatchSuccessWithoutDeletion) +{ + NoOpConditionalBatchDeleteBackend backend; + auto requests = makeRequests(backend); + auto op = requests.admit(); + + EXPECT_FALSE(runCapabilityProbe(op, "p/.cas_probe").conditional_batch_delete_supported); + EXPECT_TRUE(op.list("p/.cas_probe", "", 10, Retry::once()).keys.empty()); +} + +TEST(CASProbe, RequiresGoneForAnAlreadyDeletedKeyInTheBatch) +{ + ReportsAbsentKeyAsRemovedBackend backend; + auto requests = makeRequests(backend); + auto op = requests.admit(); + + EXPECT_FALSE(runCapabilityProbe(op, "p/.cas_probe").conditional_batch_delete_supported); +} + TEST(CASProbe, FailsClosedOnNonEnforcingDelete) { auto b = std::make_shared(); @@ -284,7 +340,7 @@ TEST(CASProbe, ReorderedProbePassesOnAllThreeDialects) auto op = requests.admit(); EXPECT_NO_THROW(runCapabilityProbe(op, "p/.cas_probe")) << "dialect " << static_cast(dialect); EXPECT_TRUE(op.list("p/.cas_probe", "", 10, Retry::once()).keys.empty()) << "dialect " << static_cast(dialect); - EXPECT_EQ(b.write_reached, 7) << "dialect " << static_cast(dialect); + EXPECT_EQ(b.write_reached, 8) << "dialect " << static_cast(dialect); EXPECT_EQ(b.remove_reached, 3) << "dialect " << static_cast(dialect); } } From 2cd032f2b6a79f484979774ba4fb2362ee35cac3 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Wed, 18 Nov 2026 16:50:20 +0100 Subject: [PATCH 23/30] Extract the CAS conditional batch delete probe Move the optional batch check into `probeConditionalBatchDelete`, where it owns its temporary objects and checks their state before cleanup. Pass the key and token from the required single-object check to exercise the already absent case in the same batch. Define chunk, batch, and bulk delete in the GC documentation. Explain that one-key chunks still use the configured I/O concurrency. Co-authored-by: Codex (GPT-6.2 Sol) Signed-off-by: Konstantin Morozov --- .../cas/architecture/garbage-collection.md | 7 + .../ContentAddressed/Backend/CasProbe.cpp | 351 +++++++++--------- .../ContentAddressed/Backend/CasRequests.h | 4 +- 3 files changed, 181 insertions(+), 181 deletions(-) diff --git a/docs/en/antalya/cas/architecture/garbage-collection.md b/docs/en/antalya/cas/architecture/garbage-collection.md index 360df9fd2d0d5..01efb7c11e698 100644 --- a/docs/en/antalya/cas/architecture/garbage-collection.md +++ b/docs/en/antalya/cas/architecture/garbage-collection.md @@ -415,6 +415,13 @@ deletes for entries a *previous* round published as `delete_pending`, processing `cas_gc_round_redelete_budget` entries per round (the excess is carried unchanged), and writes the forensic outcome logs. +In the GC code and metrics, a **chunk** is the group of keys handled by one delete job. A chunk can +contain a single key. A **batch** is one storage request that carries multiple keys, such as S3 +`DeleteObjects`. The mount probe checks whether the backend supports conditional batches, and GC +uses one-key chunks when that capability is unavailable. Those jobs can still run concurrently +according to `cas_gc_io_concurrency`. **Bulk delete** names the overall feature, including the +sequence of requests needed to process many keys. + - **Runs on:** fold path only - **Reads:** one `HEAD` per `redelete` entry — the persisted condemned token cannot itself be a precondition, so the round observes the blob first, which also settles the absent case without diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasProbe.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasProbe.cpp index cd31bb04c3303..4383d55a0b4b0 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasProbe.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasProbe.cpp @@ -1,5 +1,8 @@ #include #include +#include + +#include namespace DB { @@ -42,6 +45,57 @@ Removal removeOrReportDeleteMarker(CasOperation & op, const String & key, const } } +/// Best-effort cleanup uses a fresh token and skips keys that are already absent. +void cleanupProbeKeys(CasOperation & op, std::initializer_list keys) noexcept +{ + try + { + for (const String * cleanup_key : keys) + { + const auto current = op.head(*cleanup_key, Retry::standard()); + if (current) + op.remove(*cleanup_key, current->etag, Retry::standard()); + } + } + catch (...) {} /// NOLINT(bugprone-empty-catch) +} + +} + +bool probeConditionalBatchDelete(CasOperation & op, const String & probe_prefix, const ConditionalRemove & absent_key) +{ + const String first_matching_key = probe_prefix + "/batch-first"; + const String replaced_key = probe_prefix + "/batch-second"; + const String second_matching_key = probe_prefix + "/batch-third"; + SCOPE_EXIT({ cleanupProbeKeys(op, {&first_matching_key, &replaced_key, &second_matching_key}); }); + + const Etag first_matching_token = std::get(op.create(first_matching_key, "probe-batch-first", Retry::standard())).etag; + const Etag stale_token = std::get(op.create(replaced_key, "probe-batch-second-v1", Retry::standard())).etag; + const String replacement_bytes = "probe-batch-second-v2"; + (void)std::get(op.replace(replaced_key, replacement_bytes, stale_token, Retry::standard())); + const Etag second_matching_token = std::get(op.create(second_matching_key, "probe-batch-third", Retry::standard())).etag; + try + { + const auto removals = op.removeManyByConditional( + {{first_matching_key, first_matching_token}, {replaced_key, stale_token}, + {second_matching_key, second_matching_token}, absent_key}, + Retry::standard(), kBulkDeleteMaxKeys); + if (removals.size() != 4 || removals[0] != Removal::Removed || removals[1] != Removal::Mismatch + || removals[2] != Removal::Removed || removals[3] != Removal::Gone) + return false; + + /// Check before cleanup, which would otherwise hide a successful response without deletion. + const auto first = op.head(first_matching_key, Retry::standard()); + const auto replacement = op.read(replaced_key, Retry::standard()); + const auto second = op.head(second_matching_key, Retry::standard()); + return !first && !second && replacement && replacement->bytes == replacement_bytes; + } + catch (const DB::Exception & e) + { + if (e.code() != DB::ErrorCodes::NOT_IMPLEMENTED) + throw; + return false; + } } CapabilityProbeResult runCapabilityProbe(CasOperation & op, const String & probe_prefix) @@ -49,197 +103,138 @@ CapabilityProbeResult runCapabilityProbe(CasOperation & op, const String & probe // Sub-directory style ("probe_prefix/token") ensures that list(probe_prefix, …) works for both the // in-memory backend (prefix match) and the LocalObjectStorage backend (directory listing). const String key = probe_prefix + "/token"; - const String first_key = probe_prefix + "/batch-first"; - const String second_key = probe_prefix + "/batch-second"; - const String third_key = probe_prefix + "/batch-third"; + SCOPE_EXIT({ cleanupProbeKeys(op, {&key}); }); - // Best-effort cleanup — runs at function exit regardless of outcome. - auto cleanup = [&]() noexcept + // ---- Step 1: create fresh -> Committed; read-after-write returns the bytes. ---- + Etag t1 = [&] { - // Skip the remove when HEAD says the key is already gone (the happy path: the battery's own - // delete already ran). `Etag` can only be minted from an actual HEAD/read observation, so - // an unconditional "delete with whatever precondition" this backend never saw is not - // constructible here — the gate below is the only way to reach `remove` at all. - try - { - for (const String * cleanup_key : {&key, &first_key, &second_key, &third_key}) - { - const auto h = op.head(*cleanup_key, Retry::standard()); - if (h) - op.remove(*cleanup_key, h->etag, Retry::standard()); - } - } - catch (...) {} /// NOLINT(bugprone-empty-catch) - }; - - CapabilityProbeResult result; - try + WriteResult r = op.create(key, "probe-v1", Retry::standard()); + if (!std::holds_alternative(r)) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: create on a fresh key did not commit — backend is unexpectedly occupied or broken"); + return std::get(r).etag; + }(); { - // ---- Step 1: create fresh -> Committed; read-after-write returns the bytes. ---- - Etag t1 = [&] - { - WriteResult r = op.create(key, "probe-v1", Retry::standard()); - if (!std::holds_alternative(r)) - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, - "CasProbe: create on a fresh key did not commit — backend is unexpectedly occupied or broken"); - return std::get(r).etag; - }(); - { - const auto g = op.read(key, Retry::standard()); - if (!g || g->bytes != "probe-v1") - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, - "CasProbe: read-after-write failed — create succeeded but the object is not readable"); - } - - // ---- Step 2: create the same key again -> Conflict; bytes intact. ---- - { - WriteResult r = op.create(key, "should-not-land", Retry::standard()); - const auto * conflict = std::get_if(&r); - if (!conflict) - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, - "CasProbe: create on an existing key was not rejected (a conflict was expected) — " - "backend does not enforce conditional create"); - const auto * seen = std::get_if(&conflict->seen); - if (!seen || seen->bytes != "probe-v1") - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, - "CasProbe: create's conflict was reported but the original bytes were clobbered — " - "backend does not enforce conditional create"); - } - - // ---- Step 3: replace against the CURRENT incarnation (t1) -> Committed; incarnation changed; - // bytes replaced. Every "wrong incarnation" step below reuses THIS key's own prior - // incarnations (never a synthetic value) — an `Etag` is minted only from an - // actual backend observation, so there is no other way to name one that is - // guaranteed wrong yet dialect-valid. ---- - Etag t2 = [&] - { - WriteResult r = op.replace(key, "probe-v2", t1, Retry::standard()); - if (!std::holds_alternative(r)) - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, - "CasProbe: replace with the correct incarnation was rejected — backend does not accept a valid overwrite"); - Etag next = std::get(r).etag; - if (next == t1) - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, - "CasProbe: replace succeeded but did not mint a new incarnation — an incarnation must " - "change on every write"); - return next; - }(); - { - const auto g = op.read(key, Retry::standard()); - if (!g || g->bytes != "probe-v2") - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, - "CasProbe: replace succeeded but the new bytes are not readable"); - } + const auto g = op.read(key, Retry::standard()); + if (!g || g->bytes != "probe-v1") + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: read-after-write failed — create succeeded but the object is not readable"); + } - // ---- Step 4: replace against t1, now STALE (the key committed to t2 in step 3) -> Conflict; - // bytes intact. ---- - { - WriteResult r = op.replace(key, "clobbered", t1, Retry::standard()); - const auto * conflict = std::get_if(&r); - if (!conflict) - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, - "CasProbe: replace with a stale incarnation was not rejected (a conflict was expected) — " - "backend does not enforce conditional overwrite"); - const auto * seen = std::get_if(&conflict->seen); - if (!seen || seen->bytes != "probe-v2") - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, - "CasProbe: replace with a stale incarnation was 'rejected' but the original bytes were clobbered"); - } + // ---- Step 2: create the same key again -> Conflict; bytes intact. ---- + { + WriteResult r = op.create(key, "should-not-land", Retry::standard()); + const auto * conflict = std::get_if(&r); + if (!conflict) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: create on an existing key was not rejected (a conflict was expected) — " + "backend does not enforce conditional create"); + const auto * seen = std::get_if(&conflict->seen); + if (!seen || seen->bytes != "probe-v1") + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: create's conflict was reported but the original bytes were clobbered — " + "backend does not enforce conditional create"); + } - // ---- Step 5: remove with a STALE incarnation (t1) -> Mismatch; the object survives. A store - // that ignores this precondition removes the object here, so this path can reach a - // delete marker exactly like step 7's — route it through the same reporter. ---- - { - const Removal d = removeOrReportDeleteMarker(op, key, t1); - if (d != Removal::Mismatch) - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, - "CasProbe: remove with a stale incarnation was not rejected (a mismatch was expected) — " - "backend does not enforce conditional deletes"); - const auto g = op.read(key, Retry::standard()); - if (!g) - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, - "CasProbe: remove with a stale incarnation was rejected (correctly) but the object was deleted anyway — " - "backend does not enforce conditional deletes"); - } + // ---- Step 3: replace against the CURRENT incarnation (t1) -> Committed; incarnation changed; + // bytes replaced. Every "wrong incarnation" step below reuses THIS key's own prior + // incarnations (never a synthetic value) — an `Etag` is minted only from an + // actual backend observation, so there is no other way to name one that is + // guaranteed wrong yet dialect-valid. ---- + Etag t2 = [&] + { + WriteResult r = op.replace(key, "probe-v2", t1, Retry::standard()); + if (!std::holds_alternative(r)) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: replace with the correct incarnation was rejected — backend does not accept a valid overwrite"); + Etag next = std::get(r).etag; + if (next == t1) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: replace succeeded but did not mint a new incarnation — an incarnation must " + "change on every write"); + return next; + }(); + { + const auto g = op.read(key, Retry::standard()); + if (!g || g->bytes != "probe-v2") + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: replace succeeded but the new bytes are not readable"); + } - // ---- Step 6: list(probe_prefix) contains the probe key (list-after-write). ---- - { - bool found = false; - op.forEachListedKey(probe_prefix, [&](const ListedKey & listed) -> bool - { - if (listed.key != key) - return true; - found = true; - return false; - }, Retry::standard()); - if (!found) - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, - "CasProbe: list-after-write failed — the probe key '{}' is not visible in the listing under prefix '{}'", - key, probe_prefix); - } + // ---- Step 4: replace against t1, now STALE (the key committed to t2 in step 3) -> Conflict; + // bytes intact. ---- + { + WriteResult r = op.replace(key, "clobbered", t1, Retry::standard()); + const auto * conflict = std::get_if(&r); + if (!conflict) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: replace with a stale incarnation was not rejected (a conflict was expected) — " + "backend does not enforce conditional overwrite"); + const auto * seen = std::get_if(&conflict->seen); + if (!seen || seen->bytes != "probe-v2") + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: replace with a stale incarnation was 'rejected' but the original bytes were clobbered"); + } - // ---- Step 7: remove with the CORRECT incarnation (t2) -> Removed; no delete marker; object - // gone; list no longer contains the key. ---- - { - const Removal d = removeOrReportDeleteMarker(op, key, t2); - if (d != Removal::Removed) - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, - "CasProbe: remove with the correct incarnation was not Removed — backend rejected a valid incarnation-exact delete"); - const auto g = op.read(key, Retry::standard()); - if (g) - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, - "CasProbe: remove succeeded (Removed) but the object is still readable — backend delete is not effective"); - bool still_listed = false; - op.forEachListedKey(probe_prefix, [&](const ListedKey & listed) -> bool - { - if (listed.key != key) - return true; - still_listed = true; - return false; - }, Retry::standard()); - if (still_listed) - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, - "CasProbe: list-after-delete failed — the deleted probe key '{}' is still visible in the listing under prefix '{}'", - key, probe_prefix); - } + // ---- Step 5: remove with a STALE incarnation (t1) -> Mismatch; the object survives. A store + // that ignores this precondition removes the object here, so this path can reach a + // delete marker exactly like step 7's — route it through the same reporter. ---- + { + const Removal d = removeOrReportDeleteMarker(op, key, t1); + if (d != Removal::Mismatch) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: remove with a stale incarnation was not rejected (a mismatch was expected) — " + "backend does not enforce conditional deletes"); + const auto g = op.read(key, Retry::standard()); + if (!g) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: remove with a stale incarnation was rejected (correctly) but the object was deleted anyway — " + "backend does not enforce conditional deletes"); + } - const Etag first_token = std::get(op.create(first_key, "probe-batch-first", Retry::standard())).etag; - const Etag second_token = std::get(op.create(second_key, "probe-batch-second-v1", Retry::standard())).etag; - const Etag second_replacement = std::get(op.replace(second_key, "probe-batch-second-v2", second_token, Retry::standard())).etag; - const Etag third_token = std::get(op.create(third_key, "probe-batch-third", Retry::standard())).etag; - try - { - /// Step 7 already deleted `key` using `t2` and verified that it was gone. - const std::vector removals = op.removeManyByConditional( - {{first_key, first_token}, {second_key, second_token}, {third_key, third_token}, {key, t2}}, - Retry::standard(), kBulkDeleteMaxKeys); - if (removals.size() == 4 && removals[0] == Removal::Removed && removals[1] == Removal::Mismatch - && removals[2] == Removal::Removed && removals[3] == Removal::Gone) - { - /// Check before cleanup, which would otherwise hide a successful response without deletion. - const auto first = op.head(first_key, Retry::standard()); - const auto second = op.read(second_key, Retry::standard()); - const auto third = op.head(third_key, Retry::standard()); - result.conditional_batch_delete_supported = !first && !third && second && second->bytes == "probe-batch-second-v2"; - } - } - catch (const DB::Exception & e) + // ---- Step 6: list(probe_prefix) contains the probe key (list-after-write). ---- + { + bool found = false; + op.forEachListedKey(probe_prefix, [&](const ListedKey & listed) -> bool { - if (e.code() != DB::ErrorCodes::NOT_IMPLEMENTED) - throw; - } + if (listed.key != key) + return true; + found = true; + return false; + }, Retry::standard()); + if (!found) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: list-after-write failed — the probe key '{}' is not visible in the listing under prefix '{}'", + key, probe_prefix); } - catch (...) + + // ---- Step 7: remove with the CORRECT incarnation (t2) -> Removed; no delete marker; object + // gone; list no longer contains the key. ---- { - // Best-effort cleanup on failure path before re-throwing. - cleanup(); - throw; + const Removal d = removeOrReportDeleteMarker(op, key, t2); + if (d != Removal::Removed) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: remove with the correct incarnation was not Removed — backend rejected a valid incarnation-exact delete"); + const auto g = op.read(key, Retry::standard()); + if (g) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: remove succeeded (Removed) but the object is still readable — backend delete is not effective"); + bool still_listed = false; + op.forEachListedKey(probe_prefix, [&](const ListedKey & listed) -> bool + { + if (listed.key != key) + return true; + still_listed = true; + return false; + }, Retry::standard()); + if (still_listed) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: list-after-delete failed — the deleted probe key '{}' is still visible in the listing under prefix '{}'", + key, probe_prefix); } - // Normal-exit cleanup (the key was already deleted in step 7). Call cleanup anyway to handle any - // partial state edge cases — it is a no-op if the key is gone. - cleanup(); - return result; + /// Step 7 verified that `key` was gone, so reuse its token for the absent entry. + return {.conditional_batch_delete_supported = probeConditionalBatchDelete(op, probe_prefix, {key, t2})}; } } diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.h index 202282a54c541..b541b17e27168 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.h @@ -35,8 +35,6 @@ namespace DB::Cas { class CasOperation; -struct CapabilityProbeResult; -CapabilityProbeResult runCapabilityProbe(CasOperation & op, const String & probe_prefix); /// TRUE when the store's own answer proves this write never applied: a malformed request, an entity /// too large, an access denial, or a credential failure. Whether a STALE CREDENTIAL explains it is not @@ -374,7 +372,7 @@ class CasOperation WriteResult readModifyWriteOnPresence(const String & key, const DecideOnMeta & decide, const Retry & policy); private: - friend CapabilityProbeResult runCapabilityProbe(CasOperation & op, const String & probe_prefix); + friend bool probeConditionalBatchDelete(CasOperation & op, const String & probe_prefix, const ConditionalRemove & absent_key); std::vector removeManyByConditional( const std::vector & removals, const Retry & policy, const size_t limit); From 0e793786ac273737fff667fa50690be0ff525cc0 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Wed, 18 Nov 2026 16:59:29 +0100 Subject: [PATCH 24/30] Restore the required CAS probe structure Keep the seven required checks and their outer cleanup flow in `runCapabilityProbe`. Call the extracted `probeConditionalBatchDelete` from the same point in that flow so the refactor remains scoped to the optional batch check. Co-authored-by: Codex (GPT-6.2 Sol) Signed-off-by: Konstantin Morozov --- .../ContentAddressed/Backend/CasProbe.cpp | 257 ++++++++++-------- 1 file changed, 137 insertions(+), 120 deletions(-) diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasProbe.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasProbe.cpp index 4383d55a0b4b0..b9ab87b79f1e9 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasProbe.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasProbe.cpp @@ -103,138 +103,155 @@ CapabilityProbeResult runCapabilityProbe(CasOperation & op, const String & probe // Sub-directory style ("probe_prefix/token") ensures that list(probe_prefix, …) works for both the // in-memory backend (prefix match) and the LocalObjectStorage backend (directory listing). const String key = probe_prefix + "/token"; - SCOPE_EXIT({ cleanupProbeKeys(op, {&key}); }); - // ---- Step 1: create fresh -> Committed; read-after-write returns the bytes. ---- - Etag t1 = [&] - { - WriteResult r = op.create(key, "probe-v1", Retry::standard()); - if (!std::holds_alternative(r)) - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, - "CasProbe: create on a fresh key did not commit — backend is unexpectedly occupied or broken"); - return std::get(r).etag; - }(); - { - const auto g = op.read(key, Retry::standard()); - if (!g || g->bytes != "probe-v1") - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, - "CasProbe: read-after-write failed — create succeeded but the object is not readable"); - } + // Best-effort cleanup — runs at function exit regardless of outcome. + auto cleanup = [&]() noexcept { cleanupProbeKeys(op, {&key}); }; - // ---- Step 2: create the same key again -> Conflict; bytes intact. ---- + CapabilityProbeResult result; + try { - WriteResult r = op.create(key, "should-not-land", Retry::standard()); - const auto * conflict = std::get_if(&r); - if (!conflict) - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, - "CasProbe: create on an existing key was not rejected (a conflict was expected) — " - "backend does not enforce conditional create"); - const auto * seen = std::get_if(&conflict->seen); - if (!seen || seen->bytes != "probe-v1") - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, - "CasProbe: create's conflict was reported but the original bytes were clobbered — " - "backend does not enforce conditional create"); - } + // ---- Step 1: create fresh -> Committed; read-after-write returns the bytes. ---- + Etag t1 = [&] + { + WriteResult r = op.create(key, "probe-v1", Retry::standard()); + if (!std::holds_alternative(r)) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: create on a fresh key did not commit — backend is unexpectedly occupied or broken"); + return std::get(r).etag; + }(); + { + const auto g = op.read(key, Retry::standard()); + if (!g || g->bytes != "probe-v1") + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: read-after-write failed — create succeeded but the object is not readable"); + } - // ---- Step 3: replace against the CURRENT incarnation (t1) -> Committed; incarnation changed; - // bytes replaced. Every "wrong incarnation" step below reuses THIS key's own prior - // incarnations (never a synthetic value) — an `Etag` is minted only from an - // actual backend observation, so there is no other way to name one that is - // guaranteed wrong yet dialect-valid. ---- - Etag t2 = [&] - { - WriteResult r = op.replace(key, "probe-v2", t1, Retry::standard()); - if (!std::holds_alternative(r)) - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, - "CasProbe: replace with the correct incarnation was rejected — backend does not accept a valid overwrite"); - Etag next = std::get(r).etag; - if (next == t1) - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, - "CasProbe: replace succeeded but did not mint a new incarnation — an incarnation must " - "change on every write"); - return next; - }(); - { - const auto g = op.read(key, Retry::standard()); - if (!g || g->bytes != "probe-v2") - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, - "CasProbe: replace succeeded but the new bytes are not readable"); - } + // ---- Step 2: create the same key again -> Conflict; bytes intact. ---- + { + WriteResult r = op.create(key, "should-not-land", Retry::standard()); + const auto * conflict = std::get_if(&r); + if (!conflict) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: create on an existing key was not rejected (a conflict was expected) — " + "backend does not enforce conditional create"); + const auto * seen = std::get_if(&conflict->seen); + if (!seen || seen->bytes != "probe-v1") + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: create's conflict was reported but the original bytes were clobbered — " + "backend does not enforce conditional create"); + } - // ---- Step 4: replace against t1, now STALE (the key committed to t2 in step 3) -> Conflict; - // bytes intact. ---- - { - WriteResult r = op.replace(key, "clobbered", t1, Retry::standard()); - const auto * conflict = std::get_if(&r); - if (!conflict) - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, - "CasProbe: replace with a stale incarnation was not rejected (a conflict was expected) — " - "backend does not enforce conditional overwrite"); - const auto * seen = std::get_if(&conflict->seen); - if (!seen || seen->bytes != "probe-v2") - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, - "CasProbe: replace with a stale incarnation was 'rejected' but the original bytes were clobbered"); - } + // ---- Step 3: replace against the CURRENT incarnation (t1) -> Committed; incarnation changed; + // bytes replaced. Every "wrong incarnation" step below reuses THIS key's own prior + // incarnations (never a synthetic value) — an `Etag` is minted only from an + // actual backend observation, so there is no other way to name one that is + // guaranteed wrong yet dialect-valid. ---- + Etag t2 = [&] + { + WriteResult r = op.replace(key, "probe-v2", t1, Retry::standard()); + if (!std::holds_alternative(r)) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: replace with the correct incarnation was rejected — backend does not accept a valid overwrite"); + Etag next = std::get(r).etag; + if (next == t1) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: replace succeeded but did not mint a new incarnation — an incarnation must " + "change on every write"); + return next; + }(); + { + const auto g = op.read(key, Retry::standard()); + if (!g || g->bytes != "probe-v2") + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: replace succeeded but the new bytes are not readable"); + } - // ---- Step 5: remove with a STALE incarnation (t1) -> Mismatch; the object survives. A store - // that ignores this precondition removes the object here, so this path can reach a - // delete marker exactly like step 7's — route it through the same reporter. ---- - { - const Removal d = removeOrReportDeleteMarker(op, key, t1); - if (d != Removal::Mismatch) - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, - "CasProbe: remove with a stale incarnation was not rejected (a mismatch was expected) — " - "backend does not enforce conditional deletes"); - const auto g = op.read(key, Retry::standard()); - if (!g) - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, - "CasProbe: remove with a stale incarnation was rejected (correctly) but the object was deleted anyway — " - "backend does not enforce conditional deletes"); - } + // ---- Step 4: replace against t1, now STALE (the key committed to t2 in step 3) -> Conflict; + // bytes intact. ---- + { + WriteResult r = op.replace(key, "clobbered", t1, Retry::standard()); + const auto * conflict = std::get_if(&r); + if (!conflict) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: replace with a stale incarnation was not rejected (a conflict was expected) — " + "backend does not enforce conditional overwrite"); + const auto * seen = std::get_if(&conflict->seen); + if (!seen || seen->bytes != "probe-v2") + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: replace with a stale incarnation was 'rejected' but the original bytes were clobbered"); + } - // ---- Step 6: list(probe_prefix) contains the probe key (list-after-write). ---- - { - bool found = false; - op.forEachListedKey(probe_prefix, [&](const ListedKey & listed) -> bool + // ---- Step 5: remove with a STALE incarnation (t1) -> Mismatch; the object survives. A store + // that ignores this precondition removes the object here, so this path can reach a + // delete marker exactly like step 7's — route it through the same reporter. ---- { - if (listed.key != key) - return true; - found = true; - return false; - }, Retry::standard()); - if (!found) - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, - "CasProbe: list-after-write failed — the probe key '{}' is not visible in the listing under prefix '{}'", - key, probe_prefix); - } + const Removal d = removeOrReportDeleteMarker(op, key, t1); + if (d != Removal::Mismatch) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: remove with a stale incarnation was not rejected (a mismatch was expected) — " + "backend does not enforce conditional deletes"); + const auto g = op.read(key, Retry::standard()); + if (!g) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: remove with a stale incarnation was rejected (correctly) but the object was deleted anyway — " + "backend does not enforce conditional deletes"); + } - // ---- Step 7: remove with the CORRECT incarnation (t2) -> Removed; no delete marker; object - // gone; list no longer contains the key. ---- - { - const Removal d = removeOrReportDeleteMarker(op, key, t2); - if (d != Removal::Removed) - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, - "CasProbe: remove with the correct incarnation was not Removed — backend rejected a valid incarnation-exact delete"); - const auto g = op.read(key, Retry::standard()); - if (g) - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, - "CasProbe: remove succeeded (Removed) but the object is still readable — backend delete is not effective"); - bool still_listed = false; - op.forEachListedKey(probe_prefix, [&](const ListedKey & listed) -> bool + // ---- Step 6: list(probe_prefix) contains the probe key (list-after-write). ---- { - if (listed.key != key) - return true; - still_listed = true; - return false; - }, Retry::standard()); - if (still_listed) - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, - "CasProbe: list-after-delete failed — the deleted probe key '{}' is still visible in the listing under prefix '{}'", - key, probe_prefix); + bool found = false; + op.forEachListedKey(probe_prefix, [&](const ListedKey & listed) -> bool + { + if (listed.key != key) + return true; + found = true; + return false; + }, Retry::standard()); + if (!found) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: list-after-write failed — the probe key '{}' is not visible in the listing under prefix '{}'", + key, probe_prefix); + } + + // ---- Step 7: remove with the CORRECT incarnation (t2) -> Removed; no delete marker; object + // gone; list no longer contains the key. ---- + { + const Removal d = removeOrReportDeleteMarker(op, key, t2); + if (d != Removal::Removed) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: remove with the correct incarnation was not Removed — backend rejected a valid incarnation-exact delete"); + const auto g = op.read(key, Retry::standard()); + if (g) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: remove succeeded (Removed) but the object is still readable — backend delete is not effective"); + bool still_listed = false; + op.forEachListedKey(probe_prefix, [&](const ListedKey & listed) -> bool + { + if (listed.key != key) + return true; + still_listed = true; + return false; + }, Retry::standard()); + if (still_listed) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "CasProbe: list-after-delete failed — the deleted probe key '{}' is still visible in the listing under prefix '{}'", + key, probe_prefix); + } + + /// Step 7 verified that `key` was gone, so reuse its token for the absent entry. + result.conditional_batch_delete_supported = probeConditionalBatchDelete(op, probe_prefix, {key, t2}); + } + catch (...) + { + // Best-effort cleanup on failure path before re-throwing. + cleanup(); + throw; } - /// Step 7 verified that `key` was gone, so reuse its token for the absent entry. - return {.conditional_batch_delete_supported = probeConditionalBatchDelete(op, probe_prefix, {key, t2})}; + // Normal-exit cleanup (the key was already deleted in step 7). Call cleanup anyway to handle any + // partial state edge cases — it is a no-op if the key is gone. + cleanup(); + return result; } } From d34bfad3d9f1415b921a4a20c1b4e67db46d40ec Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Wed, 18 Nov 2026 17:13:59 +0100 Subject: [PATCH 25/30] Verify CAS conditional batch deletion on RustFS Run identical drop workloads on two isolated CAS disks with chunk limits of one and two. Let the mount probe enable conditional batching, verify logical chunk counts and S3 delete requests per round, and list RustFS to confirm that both workloads reclaim all blob bodies. Verified locally with RustFS 1.0.6: six bodies required six requests with limit one and three requests with limit two. Co-authored-by: Codex (GPT-6.2 Sol) Signed-off-by: Konstantin Morozov --- .../configs/storage_conf.xml | 26 ++++++ .../test.py | 85 +++++++++++-------- 2 files changed, 74 insertions(+), 37 deletions(-) diff --git a/tests/integration/test_cas_gc_conditional_batch_delete/configs/storage_conf.xml b/tests/integration/test_cas_gc_conditional_batch_delete/configs/storage_conf.xml index efb13348faabc..66f2ffef21a0c 100644 --- a/tests/integration/test_cas_gc_conditional_batch_delete/configs/storage_conf.xml +++ b/tests/integration/test_cas_gc_conditional_batch_delete/configs/storage_conf.xml @@ -13,8 +13,27 @@ clickhouse 1 3600 + 1 + + 1 2 + + object_storage + s3 + cas + 30 + 10000 + itest-cas-gc-conditional-singleton + http://rustfs1:11121/test/cas_gc_conditional_singleton/ + clickhouse + clickhouse + 1 + 3600 + 1 + 1 + 1 + @@ -24,6 +43,13 @@ + + +
+ disk_cas_gc_conditional_singleton +
+
+
diff --git a/tests/integration/test_cas_gc_conditional_batch_delete/test.py b/tests/integration/test_cas_gc_conditional_batch_delete/test.py index 5c866f339f32d..d618da4f75b4c 100644 --- a/tests/integration/test_cas_gc_conditional_batch_delete/test.py +++ b/tests/integration/test_cas_gc_conditional_batch_delete/test.py @@ -4,10 +4,6 @@ cluster = ClickHouseCluster(__file__) -STORAGE_POLICY = "cas_gc_conditional_batch" -DISK_NAME = "disk_cas_gc_conditional_batch" -BLOBS_PREFIX = "cas_gc_conditional_batch/blobs/" -CHUNK_KEYS = 2 NUM_INSERTS = 6 MAX_GC_ROUNDS = 12 @@ -27,39 +23,40 @@ def start_cluster(): cluster.shutdown() -def list_blob_bodies(): +def list_blob_bodies(pool_prefix): return { o.object_name for o in cluster.rustfs_client.list_objects( - cluster.rustfs_bucket, BLOBS_PREFIX, recursive=True + cluster.rustfs_bucket, f"{pool_prefix}/blobs/", recursive=True ) if not o.object_name.endswith(".meta") } -def latest_gc_round(node): +def latest_gc_round(node, disk_name): node.query("SYSTEM FLUSH LOGS") return int( node.query( - f"SELECT max(round) FROM system.cas_gc_log WHERE disk_name = '{DISK_NAME}'" + f"SELECT max(round) FROM system.cas_gc_log WHERE disk_name = '{disk_name}'" ).strip() ) -def gc_pending_delete_rows(node, after_round): +def gc_pending_delete_rows(node, disk_name, after_round): node.query("SYSTEM FLUSH LOGS") rows = ( node.query( - "SELECT p.phase_metrics['redeleted'], p.phase_metrics['delete_chunks'], " + "SELECT f.round, p.phase_metrics['redeleted'], p.phase_metrics['deleted'], " + "p.phase_metrics['delete_chunks'], p.ProfileEvents['DiskS3DeleteObjects'], " "p.phase_metrics['read_blob_jobs_failed'], " "p.phase_metrics['delete_chunk_jobs_failed'], " "p.phase_metrics['read_blob_job_schedule_failures'], " "p.phase_metrics['delete_chunk_job_schedule_failures'] " "FROM system.cas_gc_log AS p INNER JOIN " - "(SELECT round_id FROM system.cas_gc_log " - f"WHERE disk_name = '{DISK_NAME}' AND event_type = 'Finish' " + "(SELECT round_id, round FROM system.cas_gc_log " + f"WHERE disk_name = '{disk_name}' AND event_type = 'Finish' AND outcome = 'Success' " f"AND round > {after_round}) AS f USING round_id " - f"WHERE p.disk_name = '{DISK_NAME}' AND p.event_type = 'Phase' " + f"WHERE p.disk_name = '{disk_name}' AND p.event_type = 'Phase' " "AND p.phase = 'pending_deletes' AND p.phase_metrics['redeleted'] > 0 " "ORDER BY p.event_time_microseconds" ) @@ -69,24 +66,15 @@ def gc_pending_delete_rows(node, after_round): return [tuple(int(x) for x in row.split("\t")) for row in rows] -def gc_redeleted_total(node, after_round): - node.query("SYSTEM FLUSH LOGS") - return int( - node.query( - "SELECT sum(entries_redeleted) FROM system.cas_gc_log " - f"WHERE disk_name = '{DISK_NAME}' AND event_type = 'Finish' " - f"AND round > {after_round}" - ).strip() - ) - - -def test_rustfs_pending_blob_deletes_use_singleton_fallback(): +def reclaim_blob_bodies(pool_prefix, chunk_keys): node = cluster.instances["node"] - table = "conditional_batch_redelete" - blobs_before = list_blob_bodies() + disk_name = f"disk_{pool_prefix}" + table = f"{pool_prefix}_redelete" + node.query(f"SYSTEM CAS GC STOP '{disk_name}'") + assert not list_blob_bodies(pool_prefix) node.query( f"CREATE TABLE {table} (k UInt64, v String) ENGINE = MergeTree ORDER BY k " - f"SETTINGS storage_policy = '{STORAGE_POLICY}'" + f"SETTINGS storage_policy = '{pool_prefix}'" ) node.query(f"SYSTEM STOP MERGES {table}") for i in range(NUM_INSERTS): @@ -96,25 +84,29 @@ def test_rustfs_pending_blob_deletes_use_singleton_fallback(): "SETTINGS max_insert_block_size = 1000" ) - created_blobs = list_blob_bodies() - blobs_before - assert len(created_blobs) > CHUNK_KEYS - first_round = latest_gc_round(node) + created_blobs = list_blob_bodies(pool_prefix) + assert len(created_blobs) > 2 + first_round = latest_gc_round(node, disk_name) node.query(f"DROP TABLE {table} SYNC") rows = [] remaining = created_blobs for _ in range(MAX_GC_ROUNDS): - node.query("SYSTEM CAS GC RUN") - rows = gc_pending_delete_rows(node, first_round) - remaining = created_blobs & list_blob_bodies() + node.query(f"SYSTEM CAS GC RUN '{disk_name}'") + rows = gc_pending_delete_rows(node, disk_name, first_round) + remaining = created_blobs & list_blob_bodies(pool_prefix) if not remaining and rows: break assert not remaining, f"GC left blob bodies in RustFS: {sorted(remaining)}" - assert rows, "pending_deletes never reported completed blob redeletes" + assert rows, f"{disk_name}: pending_deletes never reported completed blob redeletes" + total_deleted = total_requests = 0 for ( + round_no, redeleted, + deleted, delete_chunks, + s3_delete_requests, read_blob_jobs_failed, delete_chunk_jobs_failed, read_blob_job_schedule_failures, @@ -124,5 +116,24 @@ def test_rustfs_pending_blob_deletes_use_singleton_fallback(): assert delete_chunk_jobs_failed == 0 assert read_blob_job_schedule_failures == 0 assert delete_chunk_job_schedule_failures == 0 - assert delete_chunks == redeleted - assert gc_redeleted_total(node, first_round) >= len(created_blobs) + # The isolated workload neither replaces nor removes blob bodies outside GC. + assert redeleted == deleted, f"{disk_name}, round {round_no}: {rows}" + expected_chunks = (deleted + chunk_keys - 1) // chunk_keys + assert delete_chunks == expected_chunks, f"{disk_name}, round {round_no}: {rows}" + assert s3_delete_requests == delete_chunks, f"{disk_name}, round {round_no}: {rows}" + total_deleted += deleted + total_requests += s3_delete_requests + assert total_deleted == len(created_blobs) + return total_deleted, total_requests + + +def test_pending_blob_deletes_use_conditional_batches(): + singleton_deleted, singleton_requests = reclaim_blob_bodies( + "cas_gc_conditional_singleton", 1 + ) + batch_deleted, batch_requests = reclaim_blob_bodies("cas_gc_conditional_batch", 2) + + # With the real mount probe gating multi-key chunks, a passed comparison proves batching + # was enabled and reached RustFS. Both disks reclaim the same size workload. + assert singleton_deleted == batch_deleted + assert batch_requests < singleton_requests From 443df08abf0242f1f20988d847321e222c11eae1 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Wed, 18 Nov 2026 18:15:09 +0100 Subject: [PATCH 26/30] Separate conditional and ordinary S3 batch delete support Use the configured key limit after the CAS mount probe confirms conditional batching. Keep the ordinary DeleteObjects capability independent of conditional request results in both directions. Classify only an explicit request-level NotImplemented response as unsupported for conditional deletes. Preserve InvalidRequest and InvalidArgument as S3 errors. Cover capability independence and a subsequent multi-key request after a runtime refusal. Verified with 71 focused unit tests and the RustFS 1.0.6 CAS integration test. Co-authored-by: Codex (GPT-6.2 Sol) Signed-off-by: Konstantin Morozov --- .../Backend/CasObjectStorageBackend.cpp | 2 +- .../ObjectStorages/IObjectStorage.h | 4 + .../ObjectStorages/S3/S3ObjectStorage.cpp | 15 ++- .../ObjectStorages/S3/S3ObjectStorage.h | 1 + .../gtest_cas_s3_bulk_delete_fallback.cpp | 96 ++++++++++++++++++- 5 files changed, 105 insertions(+), 13 deletions(-) diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasObjectStorageBackend.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasObjectStorageBackend.cpp index 674e3e90d930c..a26173b4aa421 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasObjectStorageBackend.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasObjectStorageBackend.cpp @@ -1044,7 +1044,7 @@ size_t ObjectStorageBackend::conditionalBatchDeleteKeyLimit() const { if (mode != Mode::Native || !conditional_batch_delete_supported.load()) return 1; - return object_storage->batchDeleteKeyLimit(); + return object_storage->configuredBatchDeleteKeyLimit(); } void ObjectStorageBackend::pinConditionalBatchDeleteSupport(const bool supported) diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h b/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h index f2f376a1fc821..81eda5739a56f 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h @@ -470,6 +470,10 @@ class IObjectStorage /// suits a storage without a batch delete. virtual size_t batchDeleteKeyLimit() const { return 1; } + /// The configured key limit for a multi-object delete request, independent of any learned + /// capability. A caller that probes conditional delete support can apply its own capability gate. + virtual size_t configuredBatchDeleteKeyLimit() const { return 1; } + /// Copy object with different attributes if required virtual void copyObject( /// NOLINT const StoredObject & object_from, diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp index a517c505ac0d6..c50348613eab7 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp @@ -695,20 +695,14 @@ std::vector S3ObjectStorage::removeObjectsIfTokensMatch { const auto & error = current_outcome.GetError(); log_all(static_cast(error.GetErrorType()), error.GetMessage()); - if (error.GetExceptionName() == "InvalidRequest" || error.GetExceptionName() == "InvalidArgument" - || error.GetExceptionName() == "NotImplemented") - { - s3_capabilities.setIsBatchDeleteSupported(false); - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "{} does not support DeleteObjects", getName()); - } + if (error.GetExceptionName() == "NotImplemented") + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "{} does not support conditional DeleteObjects", getName()); throw S3Exception( PreformattedMessage::create("{} while conditionally removing objects from S3", error.GetMessage()), error.GetErrorType(), error.GetExceptionName()); } return current_outcome; }); - s3_capabilities.setIsBatchDeleteSupported(true); - std::unordered_set expected_keys; expected_keys.reserve(objects.size()); for (const ConditionalRemoveObject & object : objects) @@ -954,6 +948,11 @@ size_t S3ObjectStorage::batchDeleteKeyLimit() const { if (const auto supported = s3_capabilities.isBatchDeleteSupported(); supported.has_value() && !*supported) return 1; + return configuredBatchDeleteKeyLimit(); +} + +size_t S3ObjectStorage::configuredBatchDeleteKeyLimit() const +{ return std::max(1, s3_settings.get()->request_settings[S3RequestSetting::objects_chunk_size_to_delete]); } diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h index 1ff12956077e8..f94e57955c451 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h @@ -145,6 +145,7 @@ class S3ObjectStorage : public IObjectStorage /// `objects_chunk_size_to_delete`, at least 1, while `DeleteObjects` is not known unsupported; 1 after. size_t batchDeleteKeyLimit() const override; + size_t configuredBatchDeleteKeyLimit() const override; void tagObjects(const StoredObjects & objects, const std::string & tag_key, const std::string & tag_value) override; diff --git a/src/Disks/tests/gtest_cas_s3_bulk_delete_fallback.cpp b/src/Disks/tests/gtest_cas_s3_bulk_delete_fallback.cpp index ae9de309d73ed..538bb51e17f7d 100644 --- a/src/Disks/tests/gtest_cas_s3_bulk_delete_fallback.cpp +++ b/src/Disks/tests/gtest_cas_s3_bulk_delete_fallback.cpp @@ -4,8 +4,10 @@ #if USE_AWS_S3 +#include #include #include +#include #include #include #include @@ -344,13 +346,18 @@ TEST(CASS3ConditionalBatchDelete, ReturnsPerKeyOutcomesInInputOrder) EXPECT_NE(server.lastBody().find("etag-second"), std::string::npos); } -TEST(CASS3ConditionalBatchDelete, ConvertsExplicitUnsupportedToNotImplemented) +TEST(CASS3ConditionalBatchDelete, UnsupportedConditionalRequestDoesNotDisableOrdinaryBatch) { (void)contextForTest(); - ScriptedS3Server server([](const Poco::Net::HTTPServerRequest &, Poco::Net::HTTPServerResponse & response) + std::atomic attempts{0}; + ScriptedS3Server server([&](const Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) { - sendBatchNotImplemented(response); + EXPECT_EQ(request.getMethod(), "POST"); + if (attempts++ == 0) + sendBatchNotImplemented(response); + else + sendDeleteObjectsResult(response, {}); }); auto storage = makeStorageForTest(server.getUrl(), DB::S3Capabilities{}); @@ -360,7 +367,11 @@ TEST(CASS3ConditionalBatchDelete, ConvertsExplicitUnsupportedToNotImplemented) {{DB::StoredObject("first"), "etag-first"}, {DB::StoredObject("second"), "etag-second"}}, DB::ObjectStorageControlRequest{}); }); - EXPECT_EQ(storage->batchDeleteKeyLimit(), 1u); + EXPECT_EQ(storage->batchDeleteKeyLimit(), 1000u); + EXPECT_NO_THROW(storage->removeObjectsIfExistUnderProfile( + {DB::StoredObject("first"), DB::StoredObject("second")}, DB::ObjectStorageControlRequest{})); + EXPECT_EQ(server.countMethod("POST"), 2u); + EXPECT_EQ(server.countMethod("DELETE"), 0u); } TEST(CASS3ConditionalBatchDelete, RefreshesExpiredCredentialsBeforeReturningFailure) @@ -630,12 +641,89 @@ void expectRefusalNamed(const std::function & remove, const std::string { EXPECT_EQ(e.getExceptionName(), name) << e.message(); } + catch (const DB::Exception & e) + { + ADD_FAILURE() << "expected an S3Exception named " << name << ", got " << e.message(); + } } const DB::StoredObjects kTwoObjects{DB::StoredObject("key-a"), DB::StoredObject("key-b")}; } +TEST(CASS3ConditionalBatchDelete, InvalidRequestAndArgumentRemainErrors) +{ + (void)contextForTest(); + for (const std::string name : {"InvalidRequest", "InvalidArgument"}) + { + SCOPED_TRACE(name); + ScriptedS3Server server([&](const Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) + { + EXPECT_EQ(request.getMethod(), "POST"); + sendSingleDeleteError(response, Poco::Net::HTTPResponse::HTTP_BAD_REQUEST, name, "invalid conditional delete input"); + }); + auto storage = makeStorageForTest(server.getUrl(), DB::S3Capabilities{}); + + expectRefusalNamed([&] + { + storage->removeObjectsIfTokensMatch( + {{DB::StoredObject("first"), "etag-first"}, {DB::StoredObject("second"), "etag-second"}}, + DB::ObjectStorageControlRequest{}); + }, name); + EXPECT_EQ(storage->batchDeleteKeyLimit(), 1000u); + EXPECT_EQ(server.countMethod("POST"), 1u); + EXPECT_EQ(server.countMethod("DELETE"), 0u); + } +} + +TEST(CASConditionalBatchDeleteBackend, RuntimeFailureDoesNotDowngradePinnedCapability) +{ + using namespace DB::Cas; + (void)contextForTest(); + std::atomic delete_attempts{0}; + ScriptedS3Server server([&](const Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) + { + if (request.getMethod() == "HEAD") + { + response.setContentLength(1); + response.set("ETag", "\"head-etag\""); + response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_OK); + response.send(); + return; + } + EXPECT_EQ(request.getMethod(), "POST"); + if (delete_attempts++ < 2) + sendBatchNotImplemented(response); + else + sendConditionalDeleteObjectsResult(response); + }); + auto storage = makeStorageForTest(server.getUrl(), DB::S3Capabilities{}, 7); + auto backend = std::make_shared(storage, ObjectStorageBackend::Mode::Native); + auto requests = tests::openRequestsForTest(backend); + auto op = requests.admit(); + EXPECT_EQ(backend->conditionalBatchDeleteKeyLimit(), 1u); + backend->pinConditionalBatchDeleteSupport(true); + ASSERT_EQ(backend->conditionalBatchDeleteKeyLimit(), 7u); + + const auto first = op.head("first", Retry::once()); + const auto second = op.head("second", Retry::once()); + ASSERT_TRUE(first && second); + const std::vector removals{{"first", first->etag}, {"second", second->etag}}; + expectNotImplemented([&] { op.removeManyByConditional(removals, Retry::once()); }); + ASSERT_EQ(backend->conditionalBatchDeleteKeyLimit(), 7u); + + expectNotImplemented([&] + { + storage->removeObjectsIfExistUnderProfile(kTwoObjects, DB::ObjectStorageControlRequest{}); + }); + EXPECT_EQ(backend->bulkDeleteKeyLimit(), 1u); + ASSERT_EQ(backend->conditionalBatchDeleteKeyLimit(), 7u); + EXPECT_EQ(op.removeManyByConditional(removals, Retry::once()), (std::vector{Removal::Removed, Removal::Mismatch})); + EXPECT_EQ(backend->bulkDeleteKeyLimit(), 1u); + EXPECT_EQ(server.countMethod("POST"), 3u); + EXPECT_EQ(server.countMethod("DELETE"), 0u); +} + TEST(CASS3BatchDelete, RefusedDeleteObjectsKeepsTheStoresErrorName) { (void)contextForTest(); From ca2203c6e26aa8cedfe257f0e9378fe4cfee0a0c Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Wed, 18 Nov 2026 22:19:41 +0100 Subject: [PATCH 27/30] Document independent S3 batch delete capabilities and RustFS coverage Co-authored-by: Codex (GPT-6) Signed-off-by: Konstantin Morozov --- .../cas/architecture/garbage-collection.md | 15 ++++++++++----- docs/en/antalya/cas/bucket-requirements.md | 2 +- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/docs/en/antalya/cas/architecture/garbage-collection.md b/docs/en/antalya/cas/architecture/garbage-collection.md index 01efb7c11e698..5ed69c17049e2 100644 --- a/docs/en/antalya/cas/architecture/garbage-collection.md +++ b/docs/en/antalya/cas/architecture/garbage-collection.md @@ -469,11 +469,16 @@ The `RoundReport::redelete_failed` field and `CASGCRetiredRedeleteFailed` event units: each failed blob from an executed job counts once, while an enqueue failure counts once per wave regardless of the number of unsubmitted blobs reported in its warning. -The project RustFS integration backend supports exact-token singleton deletes but does not implement -AWS per-object `ETag` conditions in `DeleteObjects`. Its mount probe therefore keeps one-key chunks; -integration coverage verifies this fallback and physical blob reclamation. Positive AWS batching is -covered by deterministic GC and scripted S3 transport tests. A live AWS run is still required to -validate the production service boundary end to end. +Ordinary S3 batch deletion and conditional batch deletion have independent capability flags. Only +the latter requires the store to enforce a separate `ETag` for each key in `DeleteObjects`. Both +paths use `objects_chunk_size_to_delete` as a size limit. A request-level `NotImplemented` from +the optional batch check leaves GC using one-key chunks. A request-level `InvalidRequest` or +`InvalidArgument` propagates and fails the mount. After a successful probe, a batch request error +follows the normal request retry policy and does not change the pinned capability. + +The integration test on RustFS 1.0.6 mounts two disks with GC chunk limits of one and two. It +checks the number of storage requests in each pending-delete phase and lists the bucket to verify +that the blob bodies have disappeared. Under `suppress_destructive` the `redelete` set is empty by construction, so nothing is deleted, and there is no graduation either (would-be graduates are carried unchanged); sparing and superseding diff --git a/docs/en/antalya/cas/bucket-requirements.md b/docs/en/antalya/cas/bucket-requirements.md index 015137b89906a..99ef36a0510c7 100644 --- a/docs/en/antalya/cas/bucket-requirements.md +++ b/docs/en/antalya/cas/bucket-requirements.md @@ -142,7 +142,7 @@ establish acceptance of Google's multipart, native-copy, and exact-delete wire b | AWS S3 | ✓ | Native `ETag`-based conditional dialect for mutable objects and exact deletion; blob publication is unconditional | | Google Cloud Storage | implementation complete; release gate pending | Generation-token dialect for mutable objects/native-token `HEAD`/exact deletion, opted into via `http_client = gcs_hmac` or `gcp_oauth`; blob publication uses ordinary copy/multipart. Real credentialed GCS groups have not run yet | | Azure Blob Storage | probably | Azure's REST API documents the equivalent conditional headers, but ClickHouse's Azure object-storage backend does not yet wire up a `CAS` conditional dialect the way the S3 and GCS paths do — untested, not validated by the capability probe | -| Other S3-compatible stores | only with enforced conditional operations | The capability probe is the actual gate: a store that silently ignores `If-None-Match`/`If-Match` (accepting and applying the write regardless) fails the probe and is refused. `RustFS` passes the required singleton conditional-operation battery and is used as the project's test backend, but it does not implement AWS per-object `ETag` conditions in `DeleteObjects`; conditional GC deletion therefore stays on one-key chunks. `Garage` was evaluated and rejected because it silently ignores conditional operations | +| Other S3-compatible stores | only with enforced conditional operations | A store that silently ignores required `If-None-Match` or `If-Match` conditions fails the mount probe. RustFS 1.0.6 passes both the required single-key checks and the optional per-object `ETag` check for `DeleteObjects`. Garage was evaluated and rejected because it silently ignores conditional operations. | The full mechanics of the two dialects — how the backend detects which one a given endpoint speaks, what the capability probe actually checks, and how exact-token deletes map onto each provider's From 978fd423e9a63bf01697b6ea9613df375a8fd5c6 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Wed, 18 Nov 2026 22:22:35 +0100 Subject: [PATCH 28/30] Verify failure handling and cleanup of the CAS conditional batch probe Co-authored-by: Codex (GPT-6) Signed-off-by: Konstantin Morozov --- src/Disks/tests/gtest_cas_probe.cpp | 80 +++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/src/Disks/tests/gtest_cas_probe.cpp b/src/Disks/tests/gtest_cas_probe.cpp index b15b716d8b9f5..67df2bcfb98ca 100644 --- a/src/Disks/tests/gtest_cas_probe.cpp +++ b/src/Disks/tests/gtest_cas_probe.cpp @@ -7,6 +7,20 @@ #include #include #include +#include + +#include "config.h" +#if USE_AWS_S3 +#include +#endif + +#include + +namespace DB::ErrorCodes +{ +extern const int NETWORK_ERROR; +extern const int NOT_IMPLEMENTED; +} using namespace DB::Cas; @@ -90,6 +104,25 @@ class ReportsAbsentKeyAsRemovedBackend : public InMemoryBackend } }; +class FailingConditionalBatchBackend : public InMemoryBackend +{ +public: + explicit FailingConditionalBatchBackend(const std::exception_ptr & failure_) : failure(failure_) + { + } + + std::vector removeManyByConditional(const std::vector &, TransportAccess &) override + { + ++batch_attempts; + std::rethrow_exception(failure); + } + + size_t batch_attempts = 0; + +private: + std::exception_ptr failure; +}; + } TEST(CASProbe, ReportsConditionalBatchCapability) @@ -124,6 +157,53 @@ TEST(CASProbe, RequiresGoneForAnAlreadyDeletedKeyInTheBatch) EXPECT_FALSE(runCapabilityProbe(op, "p/.cas_probe").conditional_batch_delete_supported); } +TEST(CASProbe, ExplicitlyUnsupportedConditionalBatchReturnsFalseAndCleansUp) +{ + FailingConditionalBatchBackend backend(std::make_exception_ptr( + DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "conditional batch delete unsupported"))); + auto requests = makeRequests(backend); + auto op = requests.admit(); + + EXPECT_FALSE(runCapabilityProbe(op, "p/.cas_probe").conditional_batch_delete_supported); + EXPECT_EQ(backend.batch_attempts, 1u); + EXPECT_TRUE(op.list("p/.cas_probe", "", 10, Retry::once()).keys.empty()); +} + +#if USE_AWS_S3 +TEST(CASProbe, ConditionalBatchAccessDeniedPropagatesAndCleansUp) +{ + FailingConditionalBatchBackend backend(std::make_exception_ptr( + DB::S3Exception("conditional batch delete denied", Aws::S3::S3Errors::ACCESS_DENIED))); + auto requests = makeRequests(backend); + auto op = requests.admit(); + + try + { + runCapabilityProbe(op, "p/.cas_probe"); + ADD_FAILURE() << "expected S3 access denial"; + } + catch (const DB::S3Exception & e) + { + EXPECT_EQ(e.getS3ErrorCode(), Aws::S3::S3Errors::ACCESS_DENIED); + } + EXPECT_EQ(backend.batch_attempts, 1u); + EXPECT_TRUE(op.list("p/.cas_probe", "", 10, Retry::once()).keys.empty()); +} +#endif + +TEST(CASProbe, ConditionalBatchTimeoutPropagatesAfterRetriesAndCleansUp) +{ + DB::Cas::tests::FakeClock clock; + auto backend = std::make_shared(std::make_exception_ptr(Poco::TimeoutException())); + CasRequests requests(backend, Fence::open(), clock.nowFn(), clock.sleepFn(), nullptr, kDefaultRequestWindowMs); + auto op = requests.admit(); + + DB::Cas::tests::expectThrowsCodeWithMessage(DB::ErrorCodes::NETWORK_ERROR, "removeManyByConditional", + [&] { runCapabilityProbe(op, "p/.cas_probe"); }); + EXPECT_GT(backend->batch_attempts, 1u); + EXPECT_TRUE(op.list("p/.cas_probe", "", 10, Retry::once()).keys.empty()); +} + TEST(CASProbe, FailsClosedOnNonEnforcingDelete) { auto b = std::make_shared(); From 807b77b89a1d987871f8101b52c3ad9772edb0a1 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Wed, 18 Nov 2026 22:25:39 +0100 Subject: [PATCH 29/30] Test conditional batch retries, GC recovery and S3 response validation Exercise a lost response after partial deletion and a concurrent blob replacement. Check that retries retain the original tokens and that an exhausted attempt leaves the published pending entries available to the next GC round. Co-authored-by: Codex (GPT-6) Signed-off-by: Konstantin Morozov --- .../tests/gtest_cas_bulk_delete_engine.cpp | 123 ++++++++++ .../gtest_cas_gc_redelete_concurrency.cpp | 223 +++++++++++++----- .../gtest_cas_s3_bulk_delete_fallback.cpp | 176 ++++++++++++-- 3 files changed, 449 insertions(+), 73 deletions(-) diff --git a/src/Disks/tests/gtest_cas_bulk_delete_engine.cpp b/src/Disks/tests/gtest_cas_bulk_delete_engine.cpp index 7605c98d02ee3..26761af7ff7bc 100644 --- a/src/Disks/tests/gtest_cas_bulk_delete_engine.cpp +++ b/src/Disks/tests/gtest_cas_bulk_delete_engine.cpp @@ -20,6 +20,12 @@ using namespace DB::Cas; using DB::Cas::tests::CountingBackend; using DB::Cas::tests::openRequestsForTest; +namespace DB::ErrorCodes +{ + extern const int CAS_DELETE_MARKER; + extern const int LOGICAL_ERROR; +} + namespace { @@ -41,6 +47,28 @@ class UnlimitedConditionalBatchBackend : public InMemoryBackend size_t conditionalBatchDeleteKeyLimit() const override { return kBulkDeleteMaxKeys + 1; } }; +class RecordingConditionalBackend : public InMemoryBackend +{ +public: + std::vector> attempts; + bool lose_first_response = false; + + std::vector removeManyByConditional( + const std::vector & removals, TransportAccess & access) override + { + attempts.push_back(removals); + if (lose_first_response && attempts.size() == 1 && removals.size() == 2) + { + const auto & first = removals[0]; + const auto & second = removals[1]; + EXPECT_EQ(remove(first.key, first.expected_token, access), RawRemoval::Removed); + EXPECT_TRUE(write(second.key, "replacement", second.expected_token, access).has_value()); + throw Poco::TimeoutException("lost conditional delete response"); + } + return InMemoryBackend::removeManyByConditional(removals, access); + } +}; + } TEST(CASBulkDeleteEngine, AFailedAttemptReissuesTheWholeChunkAndAlreadyDeletedKeysAreSuccess) @@ -106,6 +134,101 @@ TEST(CASConditionalBatchDeleteEngine, ReturnsMixedOutcomesInInputOrder) EXPECT_EQ(replaced->bytes, "v2"); } +TEST(CASConditionalBatchDeleteEngine, PartialSuccessRetryKeepsOriginalTokens) +{ + auto backend = std::make_shared(); + backend->pinConditionalBatchDeleteSupport(true); + backend->lose_first_response = true; + DB::Cas::tests::FakeClock clock; + CasRequests requests(backend, Fence::open(), clock.nowFn(), clock.sleepFn(), nullptr, kDefaultRequestWindowMs); + auto op = requests.admit(); + const auto first = std::get(op.create("a", "first", Retry::once())).etag; + const auto second = std::get(op.create("b", "second", Retry::once())).etag; + + const auto results = op.removeManyByConditional({{"a", first}, {"b", second}}, Retry::standard()); + + EXPECT_EQ(results, (std::vector{Removal::Gone, Removal::Mismatch})); + ASSERT_EQ(backend->attempts.size(), 2u); + for (const auto & attempt : backend->attempts) + { + ASSERT_EQ(attempt.size(), 2u); + EXPECT_EQ(attempt[0].key, "a"); + EXPECT_EQ(attempt[0].expected_token, PersistedEtag::capture(first).value); + EXPECT_EQ(attempt[1].key, "b"); + EXPECT_EQ(attempt[1].expected_token, PersistedEtag::capture(second).value); + } + EXPECT_FALSE(op.head("a", Retry::once())); + const auto replacement = op.read("b", Retry::once()); + ASSERT_TRUE(replacement); + EXPECT_EQ(replacement->bytes, "replacement"); +} + +TEST(CASConditionalBatchDeleteEngine, DeleteMarkerFailsTheChunk) +{ + auto backend = std::make_shared(); + backend->pinConditionalBatchDeleteSupport(true); + auto requests = openRequestsForTest(backend); + auto op = requests.admit(); + const auto first = std::get(op.create("a", "body", Retry::once())).etag; + const auto second = std::get(op.create("b", "body", Retry::once())).etag; + ASSERT_EQ(op.remove("a", first, Retry::once()), Removal::Removed); + backend->setSimulateDeleteMarkers(true); + + DB::Cas::tests::expectThrowsCode(DB::ErrorCodes::CAS_DELETE_MARKER, [&] + { + op.removeManyByConditional({{"a", first}, {"b", second}}, Retry::once()); + }); +} + +TEST(CASConditionalBatchDeleteEngine, EmptyInputMakesNoRequestAndAFullChunkMakesOne) +{ + auto backend = std::make_shared(); + backend->pinConditionalBatchDeleteSupport(true); + auto requests = openRequestsForTest(backend); + auto op = requests.admit(); + EXPECT_TRUE(op.removeManyByConditional({}, Retry::once()).empty()); + EXPECT_TRUE(backend->attempts.empty()); + + std::vector chunk; + for (size_t i = 0; i < kBulkDeleteMaxKeys; ++i) + { + const String key = std::to_string(i); + chunk.push_back({key, std::get(op.create(key, "body", Retry::once())).etag}); + } + EXPECT_EQ(op.removeManyByConditional(chunk, Retry::once()), std::vector(chunk.size(), Removal::Removed)); + ASSERT_EQ(backend->attempts.size(), 1u); + EXPECT_EQ(backend->attempts.front().size(), chunk.size()); +} + +#if defined(DEBUG_OR_SANITIZER_BUILD) +TEST(CASConditionalBatchDeleteEngineDeathTest, RejectsDuplicateKeysAndForeignTokensBeforeSending) +#else +TEST(CASConditionalBatchDeleteEngine, RejectsDuplicateKeysAndForeignTokensBeforeSending) +#endif +{ + auto backend = std::make_shared(); + backend->pinConditionalBatchDeleteSupport(true); + auto requests = openRequestsForTest(backend); + auto op = requests.admit(); + auto other_requests = openRequestsForTest(std::make_shared()); + auto other_op = other_requests.admit(); + const auto token = std::get(op.create("a", "body", Retry::once())).etag; + const auto foreign = std::get(other_op.create("b", "body", Retry::once())).etag; + const ConditionalRemove first{"a", token}; + for (const auto & invalid : {first, ConditionalRemove{"b", token}, ConditionalRemove{"b", foreign}}) + { + SCOPED_TRACE(invalid.expected_token.render() + " for " + invalid.key); + const std::vector chunk{first, invalid}; +#if defined(DEBUG_OR_SANITIZER_BUILD) + EXPECT_DEATH({ op.removeManyByConditional(chunk, Retry::once()); }, "duplicate key|cannot be the precondition"); +#else + DB::Cas::tests::expectThrowsCode(DB::ErrorCodes::LOGICAL_ERROR, [&] { op.removeManyByConditional(chunk, Retry::once()); }); +#endif + } + EXPECT_TRUE(backend->attempts.empty()); + EXPECT_TRUE(op.head("a", Retry::once())); +} + TEST(CASBulkDeleteEngine, ExactlyOneThousandKeysIsOneRequest) { auto backend = std::make_shared(); diff --git a/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp b/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp index d8b4ba40546b0..95c3912ccdd62 100644 --- a/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp +++ b/src/Disks/tests/gtest_cas_gc_redelete_concurrency.cpp @@ -36,6 +36,7 @@ extern const Event CASGCRetiredRedeleteFailed; namespace DB::ErrorCodes { extern const int CANNOT_SCHEDULE_TASK; +extern const int NETWORK_ERROR; } using namespace DB::Cas; @@ -126,7 +127,7 @@ class RedeleteDiagnosticsBackend : public InMemoryBackend public: std::set blob_keys; std::function read_blob_hook; - std::function &)> delete_chunk_hook; + std::function &, TransportAccess &)> delete_chunk_hook; std::optional head(const String & key, TransportAccess & access) override { @@ -146,7 +147,7 @@ class RedeleteDiagnosticsBackend : public InMemoryBackend LOG_WARNING(logger, "Test delete chunk started"); SCOPE_EXIT({ LOG_WARNING(logger, "Test delete chunk finished"); }); - delete_chunk_hook(removals); + delete_chunk_hook(removals, access); return InMemoryBackend::removeManyByConditional(removals, access); } @@ -626,53 +627,6 @@ TEST(CASGCRedeleteConcurrency, SameOutcomesAtConcurrencyOneAndFour) namespace { -class RemoveRaceBackend : public InMemoryBackend -{ -public: - void armAgainstOtherThreads(String key) - { - raced_key = std::move(key); - owner = std::this_thread::get_id(); - armed.store(true); - } - - bool racedOnWorker() const { return raced_on_worker.load(); } - - bool replacementCommitted() const { return replacement_committed.load(); } - - RawRemoval remove(const String & key, const String & expected_value, TransportAccess & access) override - { - injectReplacement(key, access); - return InMemoryBackend::remove(key, expected_value, access); - } - - std::vector removeManyByConditional( - const std::vector & removals, TransportAccess & access) override - { - for (const RawConditionalRemove & removal : removals) - injectReplacement(removal.key, access); - return InMemoryBackend::removeManyByConditional(removals, access); - } - -private: - void injectReplacement(const String & key, TransportAccess & access) - { - if (armed.load() && key == raced_key && armed.exchange(false)) - { - raced_on_worker.store(std::this_thread::get_id() != owner); - const std::optional current = InMemoryBackend::read(key, access); - if (current) - replacement_committed.store(InMemoryBackend::write(key, current->bytes, current->value, access).has_value()); - } - } - - String raced_key; - std::thread::id owner; - std::atomic armed{false}; - std::atomic raced_on_worker{false}; - std::atomic replacement_committed{false}; -}; - BlobRef blobRefOf(uint64_t blob) { return BlobRef{BlobHashAlgo::CityHash128, BlobDigest::fromU128(blobHash(blob))}; @@ -759,18 +713,45 @@ TEST(CASGCRedeleteConcurrency, ReplacedBlobInsideParallelBatchSurvivesWhileSibli TEST(CASGCRedeleteConcurrency, BlobReplacedBetweenHeadAndDeleteSurvivesWhileSiblingsDelete) { - auto backend = std::make_shared(); - auto store = openPoolWithIoConcurrency(backend, 4); + auto backend = std::make_shared(); + backend->pinConditionalBatchDeleteSupport(true); + auto store = openPoolWithIoConcurrency(backend, 4, 1, 0, {}, std::nullopt, 2); Gc gc(store, kGc); publishThenDrop(*backend, store, gc); ASSERT_NO_FATAL_FAILURE(driveUntilAllGraduated(*backend, gc, store, kReplacedBlob)); - backend->armAgainstOtherThreads(blobKeyOf(*store, kReplacedBlob)); + const String raced_key = blobKeyOf(*store, kReplacedBlob); + const auto blob_keys = allBlobKeys(*store); + const auto caller = std::this_thread::get_id(); + std::optional replacement_token; + backend->delete_chunk_hook = [&](const std::vector & removals, TransportAccess & access) + { + for (const auto & removal : removals) + { + if (removal.key != raced_key) + continue; + EXPECT_NE(std::this_thread::get_id(), caller); + ASSERT_EQ(removals.size(), 2u); + EXPECT_TRUE(std::any_of(removals.begin(), removals.end(), [&](const auto & sibling) + { + return sibling.key != raced_key && blob_keys.contains(sibling.key); + })); + const auto original = backend->read(raced_key, access); + ASSERT_TRUE(original); + const auto written = backend->write(raced_key, original->bytes, removal.expected_token, access); + ASSERT_TRUE(written.has_value()); + replacement_token = *written; + } + }; const RoundReport rep = runRegularRoundReclaiming(gc); + backend->delete_chunk_hook = {}; store->renewWatermarkOnce(); - EXPECT_TRUE(backend->replacementCommitted()) << "the race never replaced the blob, so the 412 path is untested"; - EXPECT_TRUE(backend->racedOnWorker()) << "the raced delete must have run on a pool worker"; + ASSERT_TRUE(replacement_token); + OperationForTest op(*backend); + const auto survivor = (*op).head(raced_key, Retry::once()); + ASSERT_TRUE(survivor); + EXPECT_EQ(PersistedEtag::capture(survivor->etag).value, *replacement_token); expectOnlyReplacedBlobSurvives(*backend, *store, rep); } @@ -933,7 +914,7 @@ TEST(CASGCRedeleteConcurrency, FailuresAreReportedAfterTheirWaveAndRethrowLowest if (key == preview.back().key) std::rethrow_exception(read_error); }; - backend->delete_chunk_hook = [&](const std::vector & removals) + backend->delete_chunk_hook = [&](const std::vector & removals, TransportAccess &) { if (removals.front().key == preview.front().key) { @@ -1392,6 +1373,138 @@ TEST(CASGCRedeleteConcurrency, BlobDeleteEventsMatchAppliedEntriesInAFailedRound EXPECT_EQ(next_round[blobKeyOf(*store, b)], b == kFaultedBlob ? "deleted" : "absent") << "blob " << b; } +TEST(CASGCRedeleteConcurrency, ExhaustedMultiKeyRetryKeepsPublishedPendingState) +{ + constexpr size_t chunk_size = 3; + auto events = std::make_shared(); + auto backend = std::make_shared(); + backend->pinConditionalBatchDeleteSupport(true); + /// Run chunks inline so expiring one request's virtual deadline cannot expire a sibling in flight. + auto store = openPoolWithIoConcurrency(backend, 1, 1, 0, events->sink(), std::nullopt, chunk_size); + const auto clock = VirtualRetryClock::installOn(store); + Gc gc(store, kGc); + ASSERT_NO_FATAL_FAILURE(publishThenDrop(*backend, store, gc)); + ASSERT_NO_FATAL_FAILURE(driveUntilAllGraduated(*backend, gc, store, kFaultedBlob)); + const auto pending = previewPendingDeletes(gc); + ASSERT_EQ(pending.size(), 2 * chunk_size); + OperationForTest op(*backend); + const auto & layout = store->layout(); + const auto read_state = [&] { return decodeGcState((*op).read(layout.gcStateKey(), Retry::once()).value().bytes); }; + const auto before = read_state(); + const auto runs_before = runsForShard(*backend, layout, 0); + ASSERT_FALSE(runs_before.empty()); + + size_t physical_attempts = 0; + std::optional replacement_token; + backend->delete_chunk_hook = [&](const std::vector & removals, TransportAccess & access) + { + ++physical_attempts; + ASSERT_EQ(removals.size(), chunk_size); + if (removals.front().key != pending[0].key) + return; + for (size_t i = 0; i < chunk_size; ++i) + { + EXPECT_EQ(removals[i].key, pending[i].key); + EXPECT_EQ(removals[i].expected_token, pending[i].token.value); + } + if (access.attemptNo() == 1) + { + /// A was deleted when the response was lost. A writer replaced B, and C is still eligible. + ASSERT_EQ(backend->remove(removals[0].key, removals[0].expected_token, access), Backend::RawRemoval::Removed); + const auto original = backend->read(removals[1].key, access); + ASSERT_TRUE(original); + const auto written = backend->write(removals[1].key, original->bytes, removals[1].expected_token, access); + ASSERT_TRUE(written.has_value()); + replacement_token = *written; + } + else + { + clock->advance(store->poolConfig().cas_request_budget.request_window_ms); + } + throw Poco::TimeoutException("lost conditional delete response"); + }; + + auto rows = std::make_shared(); + gc.setPhaseSink(recordPendingDeletes(rows)); + const auto failed_before = failedCounter(); + RoundReport progress; + expectThrowsCodeWithMessage(DB::ErrorCodes::NETWORK_ERROR, "policy deadline after 2 attempt(s)", [&] + { + gc.runRegularRound({}, /*allow_steal*/ true, UniversePolicy::Authoritative, &progress); + }); + backend->delete_chunk_hook = {}; + + ASSERT_TRUE(replacement_token); + EXPECT_EQ(physical_attempts, 3u); + ASSERT_EQ(rows->rows.size(), 1u); + EXPECT_EQ(rows->total("delete_chunks"), 2u); + EXPECT_EQ(rows->total("delete_chunk_jobs_failed"), 1u); + EXPECT_EQ(progress.redelete_failed, chunk_size); + EXPECT_EQ(failedCounter() - failed_before, chunk_size); + EXPECT_EQ(progress.redeleted, chunk_size); + /// GC copies per-outcome totals into the report after writing the outcome logs. This round threw before that step. + EXPECT_EQ(progress.deleted, 0u); + EXPECT_EQ(progress.absent, 0u); + EXPECT_EQ(progress.replaced, 0u); + + const auto after = read_state(); + EXPECT_EQ(after.round, before.round); + EXPECT_EQ(after.snap_generation, before.snap_generation); + EXPECT_EQ(after.snap_attempt, before.snap_attempt); + EXPECT_EQ(runsForShard(*backend, layout, 0), runs_before); + const auto still_pending = previewPendingDeletes(gc); + ASSERT_EQ(still_pending.size(), pending.size()); + const auto [reported, count] = events->take(); + EXPECT_EQ(count, chunk_size); + std::map expected_events; + for (size_t i = 0; i < pending.size(); ++i) + { + const auto & entry = pending[i]; + EXPECT_EQ(still_pending[i].key, entry.key); + EXPECT_EQ(still_pending[i].token.value, entry.token.value); + const auto body = (*op).head(entry.key, Retry::once()); + EXPECT_EQ(body.has_value(), i == 1 || i == 2) << entry.key; + EXPECT_EQ((*op).head(layout.blobMetaKey(entry.ref), Retry::once()).has_value(), i < chunk_size) << entry.key; + if (i >= chunk_size) + expected_events.emplace(entry.key, "deleted"); + } + EXPECT_EQ(reported, expected_events); + + rows->rows.clear(); + store->renewWatermarkOnce(); + const RoundReport next = runRegularRoundReclaiming(gc); + EXPECT_EQ(next.redeleted, pending.size()); + EXPECT_EQ(next.absent, chunk_size + 1); + EXPECT_EQ(next.replaced, 1u); + EXPECT_EQ(next.deleted, 1u); + EXPECT_EQ(next.redelete_failed, 0u); + EXPECT_EQ(rows->total("delete_chunks"), 1u); + EXPECT_EQ(read_state().round, before.round + 1); + EXPECT_TRUE(previewPendingDeletes(gc).empty()); + expected_events.clear(); + for (size_t i = 0; i < pending.size(); ++i) + { + const auto & entry = pending[i]; + const auto body = (*op).head(entry.key, Retry::once()); + if (i == 1) + { + ASSERT_TRUE(body); + EXPECT_EQ(PersistedEtag::capture(body->etag).value, *replacement_token); + } + else + { + EXPECT_FALSE(body) << entry.key; + } + EXPECT_EQ((*op).head(layout.blobMetaKey(entry.ref), Retry::once()).has_value(), i == 1) << entry.key; + expected_events.emplace(entry.key, "absent"); + } + expected_events.at(pending[1].key) = "replaced"; + expected_events.at(pending[2].key) = "deleted"; + const auto [recovered, recovered_count] = events->take(); + EXPECT_EQ(recovered_count, pending.size()); + EXPECT_EQ(recovered, expected_events); +} + TEST(CASGCRedeleteConcurrency, OutcomeBudgetCapsAuditRowsButNotDeletes) { constexpr uint64_t budget = 3; diff --git a/src/Disks/tests/gtest_cas_s3_bulk_delete_fallback.cpp b/src/Disks/tests/gtest_cas_s3_bulk_delete_fallback.cpp index 538bb51e17f7d..5f8a563748905 100644 --- a/src/Disks/tests/gtest_cas_s3_bulk_delete_fallback.cpp +++ b/src/Disks/tests/gtest_cas_s3_bulk_delete_fallback.cpp @@ -39,6 +39,7 @@ namespace DB::ErrorCodes { +extern const int LOGICAL_ERROR; extern const int NOT_IMPLEMENTED; } @@ -227,14 +228,16 @@ std::shared_ptr makeStorageForTest( const std::string & endpoint, const DB::S3Capabilities & capabilities, std::optional objects_chunk_size_to_delete = {}, - const DB::S3ObjectStorage::S3CredentialsRefreshCallback & credentials_refresh_callback = {}) + const DB::S3ObjectStorage::S3CredentialsRefreshCallback & credentials_refresh_callback = {}, + unsigned int max_retries = 0) { - DB::RemoteHostFilter remote_host_filter; + /// The configuration retains this reference when the storage clones its single-attempt client. + static const DB::RemoteHostFilter remote_host_filter; DB::S3::PocoHTTPClientConfiguration cfg = DB::S3::ClientFactory::instance().createClientConfiguration( "us-east-1", remote_host_filter, /* s3_max_redirects = */ 100, - DB::S3::PocoHTTPClientConfiguration::RetryStrategy{.max_retries = 0}, + DB::S3::PocoHTTPClientConfiguration::RetryStrategy{.max_retries = max_retries}, /* s3_slow_all_threads_after_network_error = */ false, /* s3_slow_all_threads_after_retryable_error = */ false, /* enable_s3_requests_logging = */ false, @@ -445,38 +448,107 @@ TEST(CASS3ConditionalBatchDelete, DoesNotTreatMissingBucketAsMissingObject) } } +TEST(CASS3ConditionalBatchDelete, ReturnsDeleteMarkerOnlyForTheMarkedKey) +{ + (void)contextForTest(); + ScriptedS3Server server([](const Poco::Net::HTTPServerRequest &, Poco::Net::HTTPServerResponse & response) + { + sendXml(response, Poco::Net::HTTPResponse::HTTP_OK, + "" + "markedtrue" + "plain" + "absentNoSuchKey" + ""); + }); + auto storage = makeStorageForTest(server.getUrl(), DB::S3Capabilities{}); + + const auto results = storage->removeObjectsIfTokensMatch( + {{DB::StoredObject("absent"), "etag-a"}, {DB::StoredObject("plain"), "etag-p"}, {DB::StoredObject("marked"), "etag-m"}}, + DB::ObjectStorageControlRequest{}); + + ASSERT_EQ(results.size(), 3u); + EXPECT_EQ(results[0].outcome, DB::ConditionalRemoveOutcome::NotFound); + EXPECT_FALSE(results[0].created_delete_marker); + EXPECT_EQ(results[1].outcome, DB::ConditionalRemoveOutcome::Removed); + EXPECT_FALSE(results[1].created_delete_marker); + EXPECT_EQ(results[2].outcome, DB::ConditionalRemoveOutcome::Removed); + EXPECT_TRUE(results[2].created_delete_marker); +} + +namespace +{ + +void expectLogicalError(const std::function & operation, const char * expected_message) +{ #if defined(DEBUG_OR_SANITIZER_BUILD) -TEST(CASS3ConditionalBatchDeleteDeathTest, RejectsDuplicateResponseKey) + EXPECT_DEATH(operation(), expected_message); #else -TEST(CASS3ConditionalBatchDelete, RejectsDuplicateResponseKey) + try + { + operation(); + FAIL() << "expected LOGICAL_ERROR: " << expected_message; + } + catch (const DB::Exception & e) + { + EXPECT_EQ(e.code(), DB::ErrorCodes::LOGICAL_ERROR) << e.message(); + EXPECT_NE(e.message().find(expected_message), std::string::npos) << e.message(); + } #endif +} + +void expectMalformedConditionalResponse(const std::string & entries, const char * expected_message) { /// Construct the server in the death-test child rather than forking its running threads. - const auto send_duplicate_response = [] + expectLogicalError([&] { (void)contextForTest(); - ScriptedS3Server server([](const Poco::Net::HTTPServerRequest &, Poco::Net::HTTPServerResponse & response) + ScriptedS3Server server([&](const Poco::Net::HTTPServerRequest &, Poco::Net::HTTPServerResponse & response) { - const std::string body = - "" - "" - "first" - "first" - "secondPreconditionFailedstale" - ""; - sendXml(response, Poco::Net::HTTPResponse::HTTP_OK, body); + sendXml(response, Poco::Net::HTTPResponse::HTTP_OK, + "" + entries + ""); }); auto storage = makeStorageForTest(server.getUrl(), DB::S3Capabilities{}); storage->removeObjectsIfTokensMatch( {{DB::StoredObject("first"), "etag-first"}, {DB::StoredObject("second"), "etag-second"}}, DB::ObjectStorageControlRequest{}); - }; + }, expected_message); +} + +} + +#if defined(DEBUG_OR_SANITIZER_BUILD) +TEST(CASS3ConditionalBatchDeleteDeathTest, RejectsDuplicateResponseKey) +#else +TEST(CASS3ConditionalBatchDelete, RejectsDuplicateResponseKey) +#endif +{ + expectMalformedConditionalResponse( + "firstfirst" + "secondPreconditionFailedstale", + "unexpected or duplicate key first"); +} + +#if defined(DEBUG_OR_SANITIZER_BUILD) +TEST(CASS3ConditionalBatchDeleteDeathTest, RejectsMissingResponseKey) +#else +TEST(CASS3ConditionalBatchDelete, RejectsMissingResponseKey) +#endif +{ + expectMalformedConditionalResponse("first", "response omitted keys"); +} + #if defined(DEBUG_OR_SANITIZER_BUILD) - EXPECT_DEATH(send_duplicate_response(), "unexpected or duplicate key first"); +TEST(CASS3ConditionalBatchDeleteDeathTest, RejectsUnexpectedResponseKey) #else - EXPECT_THROW(send_duplicate_response(), DB::Exception); +TEST(CASS3ConditionalBatchDelete, RejectsUnexpectedResponseKey) #endif +{ + for (const std::string entry : {"foreign", "foreignNoSuchKey"}) + { + SCOPED_TRACE(entry); + expectMalformedConditionalResponse("first" + entry, "unexpected or duplicate key foreign"); + } } TEST(S3BulkDeleteFallback, UnsupportedBatchReplyRecordsCapabilityFalseAndThrowsNotImplemented) @@ -651,6 +723,74 @@ const DB::StoredObjects kTwoObjects{DB::StoredObject("key-a"), DB::StoredObject( } +TEST(CASS3ConditionalBatchDelete, SingleAttemptSendsCallerAttemptNumberWithoutRetrying) +{ + (void)contextForTest(); + ScriptedS3Server server([](const Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) + { + EXPECT_EQ(request.getMethod(), "POST"); + EXPECT_EQ(request.get("clickhouse-request", ""), "attempt=7"); + sendBatchInternalError(response); + }); + auto storage = makeStorageForTest(server.getUrl(), DB::S3Capabilities{}, {}, {}, /*max_retries=*/2); + + expectRefusalNamed([&] + { + storage->removeObjectsIfTokensMatch( + {{DB::StoredObject("first"), "etag-first"}, {DB::StoredObject("second"), "etag-second"}}, + {.profile = DB::ObjectStorageRetryProfile::SingleAttempt, .attempt_number = 7}); + }, "InternalError"); + EXPECT_EQ(server.countMethod("POST"), 1u); + EXPECT_EQ(server.countMethod("DELETE"), 0u); +} + +#if defined(DEBUG_OR_SANITIZER_BUILD) +TEST(CASConditionalBatchDeleteBackendDeathTest, RejectsMalformedRawTokenBeforeSendingBatch) +#else +TEST(CASConditionalBatchDeleteBackend, RejectsMalformedRawTokenBeforeSendingBatch) +#endif +{ + using namespace DB::Cas; + + /// Inject the wildcard after admission, at the raw boundary where a backend decorator can supply a bad token. + class MalformedTokenBackend final : public InMemoryBackend + { + public: + explicit MalformedTokenBackend(ObjectStorageBackend & native_) : native(native_) + { + } + + std::vector removeManyByConditional( + const std::vector & removals, TransportAccess & access) override + { + auto malformed = removals; + malformed.back().expected_token = "*"; + return native.removeManyByConditional(malformed, access); + } + + private: + ObjectStorageBackend & native; + }; + + expectLogicalError([] + { + (void)contextForTest(); + ScriptedS3Server server([](const Poco::Net::HTTPServerRequest &, Poco::Net::HTTPServerResponse & response) + { + ADD_FAILURE() << "malformed batch reached S3"; + sendConditionalDeleteObjectsResult(response); + }); + ObjectStorageBackend native(makeStorageForTest(server.getUrl(), DB::S3Capabilities{}), ObjectStorageBackend::Mode::Native); + auto backend = std::make_shared(native); + backend->pinConditionalBatchDeleteSupport(true); + auto requests = tests::openRequestsForTest(backend); + auto op = requests.admit(); + const auto first = std::get(op.create("first", "a", Retry::once())).etag; + const auto second = std::get(op.create("second", "b", Retry::once())).etag; + op.removeManyByConditional({{"first", first}, {"second", second}}, Retry::once()); + }, "malformed conditional removal token for second"); +} + TEST(CASS3ConditionalBatchDelete, InvalidRequestAndArgumentRemainErrors) { (void)contextForTest(); From 487730e49ffb96d2615713d78a62eb18a9f47699 Mon Sep 17 00:00:00 2001 From: Konstantin Morozov Date: Fri, 20 Nov 2026 13:53:23 +0100 Subject: [PATCH 30/30] Check each invalid conditional delete against its own diagnostic The Google Test simple regex matcher rejects alternation. Check the duplicate key, wrong key, and foreign backend separately, including the keys and backend IDs in the precondition diagnostics. Enable abort on logical errors in the death-test child so the matcher also runs in release builds, route its fatal diagnostic to stderr, and keep checking the exception in the release parent. https://github.com/Altinity/ClickHouse/actions/runs/44406205665/job/135198595553 Co-authored-by: Codex (GPT-6) Signed-off-by: Konstantin Morozov --- .../tests/gtest_cas_bulk_delete_engine.cpp | 39 +++++++++++++------ 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/src/Disks/tests/gtest_cas_bulk_delete_engine.cpp b/src/Disks/tests/gtest_cas_bulk_delete_engine.cpp index 26761af7ff7bc..426f11e645e1f 100644 --- a/src/Disks/tests/gtest_cas_bulk_delete_engine.cpp +++ b/src/Disks/tests/gtest_cas_bulk_delete_engine.cpp @@ -4,7 +4,11 @@ #include #include #include +#include #include +#include +#include +#include #include /// The engine sends one chunk of at most 1000 write-once keys as one request under the ordinary @@ -200,11 +204,7 @@ TEST(CASConditionalBatchDeleteEngine, EmptyInputMakesNoRequestAndAFullChunkMakes EXPECT_EQ(backend->attempts.front().size(), chunk.size()); } -#if defined(DEBUG_OR_SANITIZER_BUILD) TEST(CASConditionalBatchDeleteEngineDeathTest, RejectsDuplicateKeysAndForeignTokensBeforeSending) -#else -TEST(CASConditionalBatchDeleteEngine, RejectsDuplicateKeysAndForeignTokensBeforeSending) -#endif { auto backend = std::make_shared(); backend->pinConditionalBatchDeleteSupport(true); @@ -215,16 +215,33 @@ TEST(CASConditionalBatchDeleteEngine, RejectsDuplicateKeysAndForeignTokensBefore const auto token = std::get(op.create("a", "body", Retry::once())).etag; const auto foreign = std::get(other_op.create("b", "body", Retry::once())).etag; const ConditionalRemove first{"a", token}; - for (const auto & invalid : {first, ConditionalRemove{"b", token}, ConditionalRemove{"b", foreign}}) + const auto expect_rejected = [&](const ConditionalRemove & invalid, const String & expected_message) { - SCOPED_TRACE(invalid.expected_token.render() + " for " + invalid.key); + SCOPED_TRACE(expected_message); const std::vector chunk{first, invalid}; -#if defined(DEBUG_OR_SANITIZER_BUILD) - EXPECT_DEATH({ op.removeManyByConditional(chunk, Retry::once()); }, "duplicate key|cannot be the precondition"); -#else - DB::Cas::tests::expectThrowsCode(DB::ErrorCodes::LOGICAL_ERROR, [&] { op.removeManyByConditional(chunk, Retry::once()); }); + EXPECT_DEATH( + { + Poco::AutoPtr channel(new Poco::ConsoleChannel(std::cerr)); + auto & logger = Poco::Logger::root(); + logger.setChannel(channel); + logger.setLevel(Poco::Message::PRIO_FATAL); + DB::abort_on_logical_error.store(true, std::memory_order_relaxed); + op.removeManyByConditional(chunk, Retry::once()); + }, + expected_message); +#if !defined(DEBUG_OR_SANITIZER_BUILD) + DB::Cas::tests::expectThrowsCodeWithMessage( + DB::ErrorCodes::LOGICAL_ERROR, expected_message, [&] { op.removeManyByConditional(chunk, Retry::once()); }); #endif - } + }; + + expect_rejected(first, "duplicate key 'a' in one chunk"); + expect_rejected({"b", token}, fmt::format( + "incarnation of 'a' observed on backend {} cannot be the precondition for 'b' on backend {}", + token.backendId(), backend->backendId())); + expect_rejected({"b", foreign}, fmt::format( + "incarnation of 'b' observed on backend {} cannot be the precondition for 'b' on backend {}", + foreign.backendId(), backend->backendId())); EXPECT_TRUE(backend->attempts.empty()); EXPECT_TRUE(op.head("a", Retry::once())); }