From 63e4289a1f229e5bd1a841e324a79f340870f800 Mon Sep 17 00:00:00 2001 From: UnamedRus Date: Sat, 8 Aug 2026 03:24:36 +0300 Subject: [PATCH 01/37] Object storage: identity cache + GetObjectAttributes to avoid per-file HEAD Opening an object went through a HEAD (getObjectMetadata) on every read to learn its size and ETag, and the result was not cached process-wide, so a scan re-HEADed every candidate file on every query (traced: ~1 S3 HeadObject per candidate file, e.g. ~1900 HEADs to read ~550 files). The HEAD is also mandatory-before-cache-lookup, because the filesystem / page / parquet-metadata caches key on (path, etag) and the etag comes from the HEAD. Introduce a process-wide identity cache and a single-request metadata fetch: * `S3::getObjectIdentity` issues one `GetObjectAttributes` request (size + ETag + multipart part sizes) and falls back to a plain HEAD if the API is unsupported, denied, or errors - safe against S3-compatible stores. The ETag is quote-normalized so the GetObjectAttributes and HEAD paths yield the same identity (they are used as cache keys). Adds the `GetObjectAttributes` wrapper to `S3::Client` (mirrors `GetObjectTagging`) and the request alias. * `ObjectStorageIdentityCache` (experimental singleton) maps path -> {etag, size, part_offsets}. `S3ObjectStorage::getObjectMetadata` consults it on the identity path (no tags requested): a hit skips the request entirely; a miss fetches via `getObjectIdentity` and populates it. The tags path and the credentials-refresh retry are unchanged. * `ObjectMetadata` gains `part_offsets` (cumulative multipart part offsets), derived from the per-part sizes; consumers use it to align reads. * `IcebergDataObjectInfo` no longer pre-populates object metadata from the manifest, so Iceberg data files also go through this single identity path and get the real ETag and multipart layout instead of a synthesized identity. New ProfileEvents: `S3GetObjectAttributes`, `ObjectStorageIdentityCacheHits`, `ObjectStorageIdentityCacheMisses`. Note: the identity path returns no user-metadata attributes (GetObjectAttributes does not provide them); callers that need them still use the tags path. The cache is an experimental singleton (not Context-managed / not SYSTEM DROP-able) and should be promoted before productionization. Signed-off-by: UnamedRus (cherry picked from commit e1b807a9bb022db747da1b9905851e7d0be4ff89) --- src/Common/ProfileEvents.cpp | 4 ++ .../ObjectStorages/IObjectStorage.h | 5 ++ .../ObjectStorageIdentityCache.cpp | 47 +++++++++++++ .../ObjectStorageIdentityCache.h | 51 ++++++++++++++ .../ObjectStorages/S3/S3ObjectStorage.cpp | 57 +++++++++++++++- src/IO/S3/Client.cpp | 6 ++ src/IO/S3/Client.h | 2 + src/IO/S3/Requests.h | 2 + src/IO/S3/getObjectInfo.cpp | 66 +++++++++++++++++++ src/IO/S3/getObjectInfo.h | 16 +++++ .../Iceberg/IcebergDataObjectInfo.cpp | 4 ++ 11 files changed, 258 insertions(+), 2 deletions(-) create mode 100644 src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp create mode 100644 src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 21a47a3f7b1c8..5753a27618bb6 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -820,6 +820,9 @@ The server successfully detected this situation and will download merged part fr M(S3ListObjects, "Number of S3 API ListObjects calls.", ValueType::Number) \ M(S3ListObjectsMicroseconds, "Time of S3 API ListObjects execution.", ValueType::Microseconds) \ M(S3HeadObject, "Number of S3 API HeadObject calls.", ValueType::Number) \ + M(S3GetObjectAttributes, "Number of S3 API GetObjectAttributes calls.", ValueType::Number) \ + M(ObjectStorageIdentityCacheHits, "Number of object-storage identity (size/etag/part-offsets) cache hits, avoiding a HEAD/GetObjectAttributes.", ValueType::Number) \ + M(ObjectStorageIdentityCacheMisses, "Number of object-storage identity cache misses that triggered a metadata request.", ValueType::Number) \ /* CA per-namespace S3 op instrumentation: one event per (namespace, op) cell of the table in */ \ /* CasInstrumentedBackend.cpp; attributes PUTs/HEADs/404s/412s/LISTs to the CA operation type. */ \ M(CASBlobPut, "Number of CAS blob PUT requests. Grows with blob uploads and indicates storage write traffic.", ValueType::Number) \ @@ -1022,6 +1025,7 @@ The server successfully detected this situation and will download merged part fr M(DiskS3CopyObject, "Number of DiskS3 API CopyObject calls.", ValueType::Number) \ M(DiskS3ListObjects, "Number of DiskS3 API ListObjects calls.", ValueType::Number) \ M(DiskS3HeadObject, "Number of DiskS3 API HeadObject calls.", ValueType::Number) \ + M(DiskS3GetObjectAttributes, "Number of DiskS3 API GetObjectAttributes calls.", ValueType::Number) \ M(DiskS3GetObjectTagging, "Number of DiskS3 API GetObjectTagging calls.", ValueType::Number) \ M(DiskS3CreateMultipartUpload, "Number of DiskS3 API CreateMultipartUpload calls.", ValueType::Number) \ M(DiskS3UploadPartCopy, "Number of DiskS3 API UploadPartCopy calls.", ValueType::Number) \ diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h b/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h index 2ced78db5b67d..6030296bc2792 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -149,6 +150,10 @@ struct ObjectMetadata bool etag_is_strong = true; ObjectAttributes tags; ObjectAttributes attributes; + /// Cumulative start offsets of the object's multipart-upload parts (part i covers + /// [part_offsets[i], part_offsets[i+1])). Populated best-effort via GetObjectAttributes; empty + /// when unknown or single-PUT. Used to align reads to part boundaries. + std::vector part_offsets; /// `etag` may be used as a content-cache key (filesystem cache, page cache, Parquet /// metadata cache) only when it is present and a strong content identifier. A weak diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp new file mode 100644 index 0000000000000..70d2fea899db8 --- /dev/null +++ b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp @@ -0,0 +1,47 @@ +#include + +#include + +namespace ProfileEvents +{ + extern const Event ObjectStorageIdentityCacheHits; + extern const Event ObjectStorageIdentityCacheMisses; +} + +namespace DB +{ + +ObjectStorageIdentityCache & ObjectStorageIdentityCache::instance() +{ + static ObjectStorageIdentityCache cache; + return cache; +} + +std::optional ObjectStorageIdentityCache::tryGet(const String & key) const +{ + std::lock_guard lock(mutex); + auto it = map.find(key); + if (it == map.end()) + { + ProfileEvents::increment(ProfileEvents::ObjectStorageIdentityCacheMisses); + return std::nullopt; + } + ProfileEvents::increment(ProfileEvents::ObjectStorageIdentityCacheHits); + return it->second; +} + +void ObjectStorageIdentityCache::set(const String & key, ObjectStorageIdentity identity) +{ + std::lock_guard lock(mutex); + if (map.size() >= max_entries) + map.clear(); + map[key] = std::move(identity); +} + +void ObjectStorageIdentityCache::clear() +{ + std::lock_guard lock(mutex); + map.clear(); +} + +} diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h new file mode 100644 index 0000000000000..60ca6cb49293f --- /dev/null +++ b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h @@ -0,0 +1,51 @@ +#pragma once + +#include + +#include +#include +#include +#include + +namespace DB +{ + +/// EXPERIMENTAL, process-wide cache of object-storage "identity": the bits needed to open an object +/// without a fresh HEAD - its size, ETag, and (if multipart-uploaded) the byte offsets of its parts. +/// +/// It exists to break the ordering that forces a HEAD before every open: object caches key on +/// (path, etag), so the etag must be known up front. Here we key by path and store the etag in the +/// value, learned once (via GetObjectAttributes / a GET response) and reused. Multipart part offsets +/// ride along so reads can be aligned to part boundaries without re-probing. +/// +/// This is a deliberately minimal singleton (plain mutex + map with a coarse size cap), meant for +/// measuring the win on this research branch. Productionizing should promote it to a Context-managed +/// CacheBase with proper eviction, server settings, and SYSTEM DROP support. +struct ObjectStorageIdentity +{ + String etag; + UInt64 size = 0; + bool is_size_known = true; + /// Cumulative start offset of each multipart part (part i covers [part_offsets[i], part_offsets[i+1])). + /// Empty when unknown or the object is a single PUT. + std::vector part_offsets; +}; + +class ObjectStorageIdentityCache +{ +public: + static ObjectStorageIdentityCache & instance(); + + std::optional tryGet(const String & key) const; + void set(const String & key, ObjectStorageIdentity identity); + void clear(); + +private: + /// Coarse cap: on overflow the whole map is cleared (experimental; a real cache would use LRU). + static constexpr size_t max_entries = 1'000'000; + + mutable std::mutex mutex; + std::unordered_map map; +}; + +} diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp index f2d67cea6728c..7ade764dcc426 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -954,11 +955,44 @@ std::optional S3ObjectStorage::tryGetObjectMetadataImpl( ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool with_tags) const { + /// When the caller only needs identity (size / etag / part offsets, not tags or user metadata) we + /// take a fast path: consult the process-wide identity cache to skip the metadata request entirely + /// on repeat opens, and on a miss fetch via GetObjectAttributes (one request for size + etag + + /// multipart part sizes, with a HEAD fallback) instead of a plain HEAD. This is what removes the + /// per-file HEAD on lake scans. The tags path is unchanged. See ObjectStorageIdentityCache. + const bool identity_only = !with_tags; + const String identity_key = uri.bucket + "/" + path; + + if (identity_only) + { + if (auto cached = ObjectStorageIdentityCache::instance().tryGet(identity_key)) + { + ObjectMetadata result; + result.size_bytes = cached->size; + result.is_size_known = cached->is_size_known; + result.etag = cached->etag; + result.part_offsets = cached->part_offsets; + /// The cache does not store Last-Modified. Leaving it at the epoch with + /// `is_last_modified_known == true` would make schema and count caches treat the object + /// as older than every cached entry. + result.is_last_modified_known = false; + return result; + } + } + auto settings_ptr = s3_settings.get(); + + auto fetch = [&](const S3::Client & c) -> S3::ObjectInfo + { + if (identity_only) + return S3::getObjectIdentity(c, uri.bucket, path, /*version_id=*/ {}); + return S3::getObjectInfo(c, uri.bucket, path, /*version_id=*/ {}, /*with_metadata=*/ true, /*with_tags=*/ with_tags); + }; + S3::ObjectInfo object_info; try { - object_info = S3::getObjectInfo(*client.get(), uri.bucket, path, /*version_id=*/ {}, /*with_metadata=*/ true, /*with_tags=*/ with_tags); + object_info = fetch(*client.get()); } catch (DB::Exception & e) { @@ -969,7 +1003,7 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool if (new_client) { client.set(std::move(new_client)); - object_info = S3::getObjectInfo(*client.get(), uri.bucket, path, /*version_id=*/ {}, /*with_metadata=*/ true, /*with_tags=*/ with_tags); + object_info = fetch(*client.get()); updated = true; } } @@ -987,6 +1021,25 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool result.etag = object_info.etag; result.tags = std::move(object_info.tags); result.attributes = object_info.metadata; + if (identity_only) + result.is_last_modified_known = object_info.last_modification_time != 0; + + /// GetObjectAttributes returns per-part sizes; convert to cumulative start offsets. + if (!object_info.part_sizes.empty()) + { + result.part_offsets.reserve(object_info.part_sizes.size()); + uint64_t offset = 0; + for (size_t part_size : object_info.part_sizes) + { + result.part_offsets.push_back(offset); + offset += part_size; + } + } + + if (identity_only) + ObjectStorageIdentityCache::instance().set( + identity_key, + ObjectStorageIdentity{result.etag, result.size_bytes, result.is_size_known, result.part_offsets}); return result; } diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp index e28948af5097c..5c86c168ede85 100644 --- a/src/IO/S3/Client.cpp +++ b/src/IO/S3/Client.cpp @@ -553,6 +553,12 @@ Model::GetBucketVersioningOutcome Client::GetBucketVersioning(GetBucketVersionin doRequest(request, [this](const Model::GetBucketVersioningRequest & req) { return GetBucketVersioning(req); })); } +Model::GetObjectAttributesOutcome Client::GetObjectAttributes(GetObjectAttributesRequest & request) const +{ + return processRequestResult( + doRequest(request, [this](const Model::GetObjectAttributesRequest & req) { return GetObjectAttributes(req); })); +} + Model::ListObjectsV2Outcome Client::ListObjectsV2(ListObjectsV2Request & request) const { return doRequestWithRetryNetworkErrors( diff --git a/src/IO/S3/Client.h b/src/IO/S3/Client.h index 8290eb0a61ead..7dfc15b406809 100644 --- a/src/IO/S3/Client.h +++ b/src/IO/S3/Client.h @@ -217,6 +217,7 @@ class Client : private Aws::S3::S3Client Model::ListObjectsV2Outcome ListObjectsV2(ListObjectsV2Request & request) const; Model::ListObjectsOutcome ListObjects(ListObjectsRequest & request) const; Model::GetObjectOutcome GetObject(GetObjectRequest & request) const; + Model::GetObjectAttributesOutcome GetObjectAttributes(GetObjectAttributesRequest & request) const; Model::AbortMultipartUploadOutcome AbortMultipartUpload(AbortMultipartUploadRequest & request) const; Model::CreateMultipartUploadOutcome CreateMultipartUpload(CreateMultipartUploadRequest & request) const; @@ -296,6 +297,7 @@ class Client : private Aws::S3::S3Client using Aws::S3::S3Client::ListObjectsV2; using Aws::S3::S3Client::ListObjects; using Aws::S3::S3Client::GetObject; + using Aws::S3::S3Client::GetObjectAttributes; using Aws::S3::S3Client::AbortMultipartUpload; using Aws::S3::S3Client::CreateMultipartUpload; diff --git a/src/IO/S3/Requests.h b/src/IO/S3/Requests.h index 1a86fbd9b8959..5430281ebe765 100644 --- a/src/IO/S3/Requests.h +++ b/src/IO/S3/Requests.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -184,6 +185,7 @@ class HeadObjectRequest: public ExtendedRequest using ListObjectsV2Request = ExtendedRequest; using ListObjectsRequest = ExtendedRequest; using GetObjectRequest = ExtendedRequest; +using GetObjectAttributesRequest = ExtendedRequest; using GetObjectTaggingRequest = ExtendedRequest; using GetBucketVersioningRequest = ExtendedRequest; diff --git a/src/IO/S3/getObjectInfo.cpp b/src/IO/S3/getObjectInfo.cpp index 23803e98b7fb1..a4c63f5d33aa5 100644 --- a/src/IO/S3/getObjectInfo.cpp +++ b/src/IO/S3/getObjectInfo.cpp @@ -5,13 +5,17 @@ #if USE_AWS_S3 +#include + namespace ProfileEvents { extern const Event S3GetObjectTagging; extern const Event S3HeadObject; extern const Event S3HeadObjectMicroseconds; + extern const Event S3GetObjectAttributes; extern const Event DiskS3GetObjectTagging; extern const Event DiskS3HeadObject; + extern const Event DiskS3GetObjectAttributes; } @@ -48,6 +52,39 @@ namespace return client.HeadObject(req); } + Aws::S3::Model::GetObjectAttributesOutcome getObjectAttributes( + const S3::Client & client, + const String & bucket, + const String & key, + const String & version_id) + { + ProfileEvents::increment(ProfileEvents::S3GetObjectAttributes); + if (client.isClientForDisk()) + ProfileEvents::increment(ProfileEvents::DiskS3GetObjectAttributes); + + S3::GetObjectAttributesRequest req; + req.SetBucket(bucket); + req.SetKey(key); + if (!version_id.empty()) + req.SetVersionId(version_id); + req.SetObjectAttributes({ + Aws::S3::Model::ObjectAttributes::ETag, + Aws::S3::Model::ObjectAttributes::ObjectSize, + Aws::S3::Model::ObjectAttributes::ObjectParts}); + + return client.GetObjectAttributes(req); + } + + /// GetObjectAttributes returns the ETag without surrounding quotes, whereas HeadObject returns it + /// quoted. Normalize (strip quotes) so identities are consistent regardless of which path produced + /// them - important because they are used as cache keys. + String stripQuotes(String s) + { + if (s.size() >= 2 && s.front() == '"' && s.back() == '"') + s = s.substr(1, s.size() - 2); + return s; + } + Aws::S3::Model::GetObjectTaggingOutcome getObjectTagging( const S3::Client & client, const String & bucket, @@ -194,6 +231,35 @@ ObjectInfo getObjectInfo( getAuthenticationErrorHint(error.GetErrorType())); } +ObjectInfo getObjectIdentity( + const S3::Client & client, + const String & bucket, + const String & key, + const String & version_id) +{ + { + Expect404ResponseScope scope; // a not-found here just falls through to HEAD below + auto outcome = getObjectAttributes(client, bucket, key, version_id); + if (outcome.IsSuccess()) + { + const auto & result = outcome.GetResult(); + ObjectInfo object_info; + object_info.size = static_cast(result.GetObjectSize()); + object_info.is_size_known = true; + object_info.etag = stripQuotes(result.GetETag()); + for (const auto & part : result.GetObjectParts().GetParts()) + object_info.part_sizes.push_back(static_cast(part.GetSize())); + return object_info; + } + /// GetObjectAttributes unsupported (many S3-compatible stores), denied (distinct IAM action), + /// or otherwise failed - fall back to a plain HEAD. Best-effort: we lose part_sizes only. + } + + ObjectInfo object_info = getObjectInfo(client, bucket, key, version_id, /*with_metadata=*/ false, /*with_tags=*/ false); + object_info.etag = stripQuotes(object_info.etag); + return object_info; +} + size_t getObjectSize( const S3::Client & client, const String & bucket, diff --git a/src/IO/S3/getObjectInfo.h b/src/IO/S3/getObjectInfo.h index d3a31260f473c..718c040c101aa 100644 --- a/src/IO/S3/getObjectInfo.h +++ b/src/IO/S3/getObjectInfo.h @@ -3,6 +3,7 @@ #include "config.h" #if USE_AWS_S3 +#include #include #include #include @@ -20,6 +21,10 @@ struct ObjectInfo String etag; ObjectAttributes tags; // Set only if getObjectInfo() is called with `with_tags = true` ObjectAttributes metadata = {}; /// Set only if getObjectInfo() is called with `with_metadata = true`. + /// Sizes of the object's multipart-upload parts (offsets = prefix sums). Set only by + /// getObjectIdentity() when the object was multipart-uploaded and GetObjectAttributes succeeded; + /// empty for single-PUT objects or when the call fell back to HEAD. + std::vector part_sizes; }; /// Ignore if object does not exist @@ -45,6 +50,17 @@ ObjectInfo getObjectInfo( bool with_metadata = false, bool with_tags = false); +/// Fetches size + etag (+ multipart part sizes) in a single GetObjectAttributes request instead of a +/// HEAD, so a caller that also wants the multipart layout gets it without a second round-trip. Falls +/// back to getObjectInfo() (HEAD) if GetObjectAttributes is unsupported, denied, or otherwise errors, +/// so it is safe against S3-compatible stores that lack the API. `part_sizes` is populated only on +/// the GetObjectAttributes path for multipart objects; it is empty on the fallback path. +ObjectInfo getObjectIdentity( + const S3::Client & client, + const String & bucket, + const String & key, + const String & version_id = {}); + ObjectAttributes getObjectTags( const S3::Client & client, const String & bucket, diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.cpp index 0f6a1b33f66bf..ea3cb97eeb645 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.cpp @@ -100,6 +100,10 @@ IcebergDataObjectInfo::IcebergDataObjectInfo( , resolved_storage(std::move(resolved_storage_)) { chassert(resolved_key_.empty() == (resolved_storage == nullptr)); + /// Object identity (size / etag / multipart part offsets) is intentionally not pre-populated from + /// the manifest. S3ObjectStorage::getObjectMetadata loads it once and keeps it in + /// ObjectStorageIdentityCache, so Iceberg data files get the real ETag used by the filesystem, + /// page, and Parquet metadata caches. Pre-populating metadata here would skip that path. } IcebergDataObjectInfo::IcebergDataObjectInfo(const RelativePathWithMetadata & path_) From ef65f443cc21be92eb286a547416c5bc207ded8d Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Mon, 16 Nov 2026 13:08:55 +0100 Subject: [PATCH 02/37] Object storage: own the identity cache from Context Replace the process-wide map with a size-bounded CacheBase created from `object_storage_identity_cache_policy`, `_size` and `_size_ratio`, resized on config reload, and cleared by `SYSTEM DROP OBJECT STORAGE IDENTITY CACHE`. S3 reads look it up from the global context and fall back to an uncached request when no context exists. Co-authored-by: Cursor --- docs/reference/statements/system.mdx | 4 ++ programs/local/LocalServer.cpp | 14 +++++ programs/server/Server.cpp | 15 +++++ src/Access/Common/AccessType.h | 1 + src/Common/CurrentMetrics.cpp | 2 + src/Common/QueryFuzzer.cpp | 1 + src/Core/ServerSettings.cpp | 3 + .../ObjectStorageIdentityCache.cpp | 44 ++++++++------- .../ObjectStorageIdentityCache.h | 56 +++++++++++++------ .../ObjectStorages/S3/S3ObjectStorage.cpp | 45 +++++++++------ src/Interpreters/Context.cpp | 43 ++++++++++++++ src/Interpreters/Context.h | 8 +++ src/Interpreters/InterpreterSystemQuery.cpp | 7 +++ src/Parsers/ASTSystemQuery.cpp | 1 + src/Parsers/ASTSystemQuery.h | 1 + src/Parsers/ParserSystemQuery.cpp | 1 + ...em_cache_on_cluster_access_types.reference | 1 + ...36_system_cache_on_cluster_access_types.sh | 1 + 18 files changed, 192 insertions(+), 56 deletions(-) diff --git a/docs/reference/statements/system.mdx b/docs/reference/statements/system.mdx index 2885ffd7bde77..5d05c519ef7b0 100644 --- a/docs/reference/statements/system.mdx +++ b/docs/reference/statements/system.mdx @@ -134,6 +134,10 @@ Clears the per-URL Confluent Schema Registry caches used by the `AvroConfluent` Clears the parquet metadata cache. +## SYSTEM CLEAR|DROP OBJECT STORAGE IDENTITY CACHE {#drop-object-storage-identity-cache} + +Clears the object-storage identity cache, which stores per-object size, ETag and multipart part offsets so that opening an object does not repeat a `HEAD` or `GetObjectAttributes` request. Its size is configured with the server settings `object_storage_identity_cache_size`, `object_storage_identity_cache_policy` and `object_storage_identity_cache_size_ratio`. Set `object_storage_identity_cache_size` to `0` to disable the cache. + ## SYSTEM CLEAR|DROP PAIMON METADATA CACHE {#drop-paimon-metadata-cache} Clears the in-memory cache of parsed Paimon metadata files (manifest lists and manifests). diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index e8d02648c755f..b1966840602ae 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -169,6 +169,9 @@ namespace ServerSetting extern const ServerSettingsString mark_cache_policy; extern const ServerSettingsUInt64 mark_cache_size; extern const ServerSettingsDouble mark_cache_size_ratio; + extern const ServerSettingsString object_storage_identity_cache_policy; + extern const ServerSettingsUInt64 object_storage_identity_cache_size; + extern const ServerSettingsDouble object_storage_identity_cache_size_ratio; extern const ServerSettingsString unique_key_bitmap_cache_policy; extern const ServerSettingsUInt64 unique_key_bitmap_cache_size_bytes; extern const ServerSettingsDouble unique_key_bitmap_cache_size_ratio; @@ -1529,6 +1532,17 @@ void LocalServer::processConfig() } global_context->setMarkCache(mark_cache_policy, mark_cache_size, mark_cache_size_ratio); + String object_storage_identity_cache_policy = server_settings[ServerSetting::object_storage_identity_cache_policy]; + size_t object_storage_identity_cache_size = server_settings[ServerSetting::object_storage_identity_cache_size]; + double object_storage_identity_cache_size_ratio = server_settings[ServerSetting::object_storage_identity_cache_size_ratio]; + if (object_storage_identity_cache_size > max_cache_size) + { + object_storage_identity_cache_size = max_cache_size; + LOG_INFO(log, "Lowered object storage identity cache size to {} because the system has limited RAM", formatReadableSizeWithBinarySuffix(object_storage_identity_cache_size)); + } + global_context->setObjectStorageIdentityCache( + object_storage_identity_cache_policy, object_storage_identity_cache_size, object_storage_identity_cache_size_ratio); + /// UNIQUE KEY delete-bitmap cache. Zero size disables. String unique_key_bitmap_cache_policy_name = server_settings[ServerSetting::unique_key_bitmap_cache_policy]; size_t unique_key_bitmap_cache_size = server_settings[ServerSetting::unique_key_bitmap_cache_size_bytes]; diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 45947fb9b9cc6..8af11d18091f1 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -320,6 +320,9 @@ namespace ServerSetting extern const ServerSettingsString mark_cache_policy; extern const ServerSettingsUInt64 mark_cache_size; extern const ServerSettingsDouble mark_cache_size_ratio; + extern const ServerSettingsString object_storage_identity_cache_policy; + extern const ServerSettingsUInt64 object_storage_identity_cache_size; + extern const ServerSettingsDouble object_storage_identity_cache_size_ratio; extern const ServerSettingsString unique_key_index_cache_policy; extern const ServerSettingsUInt64 unique_key_index_cache_size_bytes; extern const ServerSettingsDouble unique_key_index_cache_size_ratio; @@ -2475,6 +2478,17 @@ try } global_context->setMarkCache(mark_cache_policy, mark_cache_size, mark_cache_size_ratio); + String object_storage_identity_cache_policy = server_settings[ServerSetting::object_storage_identity_cache_policy]; + size_t object_storage_identity_cache_size = server_settings[ServerSetting::object_storage_identity_cache_size]; + double object_storage_identity_cache_size_ratio = server_settings[ServerSetting::object_storage_identity_cache_size_ratio]; + if (object_storage_identity_cache_size > max_cache_size) + { + object_storage_identity_cache_size = max_cache_size; + LOG_INFO(log, "Lowered object storage identity cache size to {} because the system has limited RAM", formatReadableSizeWithBinarySuffix(object_storage_identity_cache_size)); + } + global_context->setObjectStorageIdentityCache( + object_storage_identity_cache_policy, object_storage_identity_cache_size, object_storage_identity_cache_size_ratio); + String unique_key_index_cache_policy_name = server_settings[ServerSetting::unique_key_index_cache_policy]; size_t unique_key_index_cache_size = server_settings[ServerSetting::unique_key_index_cache_size_bytes]; double unique_key_index_cache_size_ratio = server_settings[ServerSetting::unique_key_index_cache_size_ratio]; @@ -3123,6 +3137,7 @@ try global_context->updateUncompressedCacheConfiguration(config(), max_cache_size_in_bytes); global_context->updateMarkCacheConfiguration(config(), max_cache_size_in_bytes); + global_context->updateObjectStorageIdentityCacheConfiguration(config(), max_cache_size_in_bytes); global_context->updateUniqueKeyIndexCacheConfiguration(config(), max_cache_size_in_bytes); global_context->updateDeleteBitmapCacheConfiguration(config(), max_cache_size_in_bytes); global_context->updatePrimaryIndexCacheConfiguration(config(), max_cache_size_in_bytes); diff --git a/src/Access/Common/AccessType.h b/src/Access/Common/AccessType.h index 1df7f90de63e7..d79f2b43dcbf2 100644 --- a/src/Access/Common/AccessType.h +++ b/src/Access/Common/AccessType.h @@ -332,6 +332,7 @@ enum class AccessType : uint16_t M(SYSTEM_DROP_PAIMON_METADATA_CACHE, "SYSTEM CLEAR PAIMON_METADATA_CACHE, SYSTEM DROP PAIMON_METADATA_CACHE", GLOBAL, SYSTEM_DROP_CACHE) \ M(SYSTEM_DROP_AVRO_SCHEMA_CACHE, "SYSTEM CLEAR AVRO SCHEMA CACHE, SYSTEM DROP AVRO SCHEMA CACHE, DROP AVRO SCHEMA CACHE", GLOBAL, SYSTEM_DROP_CACHE) \ M(SYSTEM_DROP_PARQUET_METADATA_CACHE, "SYSTEM DROP PARQUET_METADATA_CACHE", GLOBAL, SYSTEM_DROP_CACHE) \ + M(SYSTEM_DROP_OBJECT_STORAGE_IDENTITY_CACHE, "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE, SYSTEM DROP OBJECT STORAGE IDENTITY CACHE", GLOBAL, SYSTEM_DROP_CACHE) \ M(SYSTEM_DROP_POINT_IN_POLYGON_CACHE, "SYSTEM CLEAR POINT IN POLYGON CACHE, SYSTEM DROP POINT IN POLYGON CACHE, DROP POINT IN POLYGON CACHE", GLOBAL, SYSTEM_DROP_CACHE) \ M(SYSTEM_DROP_PUFFIN_FILES_CACHE, "SYSTEM DROP PUFFIN_FILES_CACHE", GLOBAL, SYSTEM_DROP_CACHE) \ M(SYSTEM_PREWARM_PRIMARY_INDEX_CACHE, "SYSTEM PREWARM PRIMARY INDEX, PREWARM PRIMARY INDEX CACHE, PREWARM PRIMARY INDEX", GLOBAL, SYSTEM_DROP_CACHE) \ diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index 5035b8cac7f33..35127b5c0289a 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -386,6 +386,8 @@ M(PaimonMetadataFilesCacheFiles, "Number of cached files in the Paimon metadata cache") \ M(ParquetMetadataCacheBytes, "Size of the Parquet metadata cache in bytes") \ M(ParquetMetadataCacheFiles, "Number of cached files in the Parquet metadata cache") \ + M(ObjectStorageIdentityCacheBytes, "Size of the object-storage identity cache in bytes") \ + M(ObjectStorageIdentityCacheCells, "Number of entries in the object-storage identity cache") \ M(PuffinFilesCacheBytes, "Size of the Puffin files cache in bytes (deletion vectors plus memoized footers)") \ M(PuffinFilesCacheFiles, "Number of cached entries in the Puffin files cache (deletion vectors plus memoized footers)") \ M(AvroSchemaCacheBytes, "Size of the Avro schema cache in bytes") \ diff --git a/src/Common/QueryFuzzer.cpp b/src/Common/QueryFuzzer.cpp index fa45aaeb1a3ab..85a6045b814ee 100644 --- a/src/Common/QueryFuzzer.cpp +++ b/src/Common/QueryFuzzer.cpp @@ -7934,6 +7934,7 @@ void QueryFuzzer::fuzz(ASTPtr & ast) Type::CLEAR_ENCRYPTION_HEADERS_CACHE, Type::CLEAR_PAIMON_METADATA_CACHE, Type::CLEAR_PARQUET_METADATA_CACHE, + Type::CLEAR_OBJECT_STORAGE_IDENTITY_CACHE, Type::CLEAR_POINT_IN_POLYGON_CACHE, /// `CLEAR_{DISK_METADATA,DISTRIBUTED,FILESYSTEM,FORMAT_SCHEMA,QUERY,SCHEMA}_CACHE` /// are deliberately absent: they carry an operand this rotation would not preserve. diff --git a/src/Core/ServerSettings.cpp b/src/Core/ServerSettings.cpp index 90b94e8383ddb..b8386d32b73d4 100644 --- a/src/Core/ServerSettings.cpp +++ b/src/Core/ServerSettings.cpp @@ -632,6 +632,9 @@ This setting can be modified at runtime and will take effect immediately. )", 0) \ DECLARE(Double, mark_cache_size_ratio, DEFAULT_MARK_CACHE_SIZE_RATIO, R"(The size of the protected queue (in case of SLRU policy) in the mark cache relative to the cache's total size.)", 0) \ DECLARE(Double, mark_cache_prewarm_ratio, 0.95, R"(The ratio of total size of mark cache to fill during prewarm.)", 0) \ + DECLARE(String, object_storage_identity_cache_policy, "SLRU", R"(Object-storage identity cache policy name (SLRU or LRU).)", 0) \ + DECLARE(UInt64, object_storage_identity_cache_size, 67108864, R"(Maximum size of the object-storage identity cache in bytes. This cache stores per-object size, ETag and multipart part offsets so that opening an object does not require a fresh HEAD/GetObjectAttributes on every read. Set to 0 to disable.)", 0) \ + DECLARE(Double, object_storage_identity_cache_size_ratio, 0.5, R"(The size of the protected queue (in case of SLRU policy) in the object-storage identity cache relative to the cache's total size.)", 0) \ DECLARE(String, unique_key_index_cache_policy, "SLRU", R"(UNIQUE KEY index cache policy name (SLRU or LRU).)", 0) \ DECLARE(UInt64, unique_key_index_cache_size_bytes, 1_GiB, R"(Maximum size (bytes) of the in-process cache for UNIQUE KEY index (SST) blocks. Set to 0 to disable the cache.)", 0) \ DECLARE(Double, unique_key_index_cache_size_ratio, 0.5, R"(The size of the protected queue (in case of SLRU policy) in the UNIQUE KEY index cache relative to the cache's total size.)", 0) \ diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp index 70d2fea899db8..7c6fc306ebaf7 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp @@ -1,6 +1,7 @@ #include #include +#include namespace ProfileEvents { @@ -8,40 +9,41 @@ namespace ProfileEvents extern const Event ObjectStorageIdentityCacheMisses; } +namespace CurrentMetrics +{ + extern const Metric ObjectStorageIdentityCacheBytes; + extern const Metric ObjectStorageIdentityCacheCells; +} + namespace DB { -ObjectStorageIdentityCache & ObjectStorageIdentityCache::instance() +ObjectStorageIdentityCache::ObjectStorageIdentityCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio) + : Base( + cache_policy, + CurrentMetrics::ObjectStorageIdentityCacheBytes, + CurrentMetrics::ObjectStorageIdentityCacheCells, + max_size_in_bytes, + /*max_count*/ 0, + size_ratio) { - static ObjectStorageIdentityCache cache; - return cache; } -std::optional ObjectStorageIdentityCache::tryGet(const String & key) const +std::optional ObjectStorageIdentityCache::tryGet(const String & key) { - std::lock_guard lock(mutex); - auto it = map.find(key); - if (it == map.end()) + if (auto mapped = Base::get(key)) { - ProfileEvents::increment(ProfileEvents::ObjectStorageIdentityCacheMisses); - return std::nullopt; + ProfileEvents::increment(ProfileEvents::ObjectStorageIdentityCacheHits); + return *mapped; } - ProfileEvents::increment(ProfileEvents::ObjectStorageIdentityCacheHits); - return it->second; -} -void ObjectStorageIdentityCache::set(const String & key, ObjectStorageIdentity identity) -{ - std::lock_guard lock(mutex); - if (map.size() >= max_entries) - map.clear(); - map[key] = std::move(identity); + ProfileEvents::increment(ProfileEvents::ObjectStorageIdentityCacheMisses); + return std::nullopt; } -void ObjectStorageIdentityCache::clear() +void ObjectStorageIdentityCache::set(const String & key, ObjectStorageIdentity identity) { - std::lock_guard lock(mutex); - map.clear(); + Base::set(key, std::make_shared(std::move(identity))); } } diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h index 60ca6cb49293f..152a05c8edeb4 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h @@ -1,26 +1,21 @@ #pragma once #include +#include -#include #include -#include #include namespace DB { -/// EXPERIMENTAL, process-wide cache of object-storage "identity": the bits needed to open an object -/// without a fresh HEAD - its size, ETag, and (if multipart-uploaded) the byte offsets of its parts. +/// Object-storage "identity": the bits needed to open an object without a fresh HEAD - its size, +/// ETag, and (if multipart-uploaded) the byte offsets of its parts. /// /// It exists to break the ordering that forces a HEAD before every open: object caches key on /// (path, etag), so the etag must be known up front. Here we key by path and store the etag in the /// value, learned once (via GetObjectAttributes / a GET response) and reused. Multipart part offsets /// ride along so reads can be aligned to part boundaries without re-probing. -/// -/// This is a deliberately minimal singleton (plain mutex + map with a coarse size cap), meant for -/// measuring the win on this research branch. Productionizing should promote it to a Context-managed -/// CacheBase with proper eviction, server settings, and SYSTEM DROP support. struct ObjectStorageIdentity { String etag; @@ -29,23 +24,48 @@ struct ObjectStorageIdentity /// Cumulative start offset of each multipart part (part i covers [part_offsets[i], part_offsets[i+1])). /// Empty when unknown or the object is a single PUT. std::vector part_offsets; + + size_t memoryUsage() const + { + return sizeof(ObjectStorageIdentity) + etag.capacity() + part_offsets.capacity() * sizeof(UInt64); + } +}; + +/// Approximate per-entry weight (bytes) for size-bounded eviction. +struct ObjectStorageIdentityWeight +{ + /// Extra bytes spent on the key string, hashmap node, list links, shared_ptr, etc. + static constexpr size_t OVERHEAD = 128; + + size_t operator()(const ObjectStorageIdentity & identity) const + { + return identity.memoryUsage() + OVERHEAD; + } }; +/// Cache of object-storage identity, keyed by object path, so opening an object does not require a +/// fresh HEAD / GetObjectAttributes on every read (a lake scan otherwise re-HEADs every candidate +/// file on every query). Backed by CacheBase (LRU/SLRU) for size-bounded eviction and thread-safety. +/// +/// It is owned by the global Context (see Context::getObjectStorageIdentityCache), sized from server +/// settings, and cleared by `SYSTEM DROP OBJECT STORAGE IDENTITY CACHE`. Low-level object-storage +/// code reaches it via Context::getGlobalContextInstance() and must tolerate a null pointer (no +/// global context yet) by simply issuing the metadata request uncached. class ObjectStorageIdentityCache + : public CacheBase, ObjectStorageIdentityWeight> { -public: - static ObjectStorageIdentityCache & instance(); + using Base = CacheBase, ObjectStorageIdentityWeight>; - std::optional tryGet(const String & key) const; - void set(const String & key, ObjectStorageIdentity identity); - void clear(); +public: + ObjectStorageIdentityCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio); -private: - /// Coarse cap: on overflow the whole map is cleared (experimental; a real cache would use LRU). - static constexpr size_t max_entries = 1'000'000; + /// Look up an identity by path. Increments the hit/miss ProfileEvents. + std::optional tryGet(const String & key); - mutable std::mutex mutex; - std::unordered_map map; + /// Insert (or overwrite) the identity for a path. + void set(const String & key, ObjectStorageIdentity identity); }; +using ObjectStorageIdentityCachePtr = std::shared_ptr; + } diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp index 7ade764dcc426..14529134ddc0c 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp @@ -956,27 +956,38 @@ std::optional S3ObjectStorage::tryGetObjectMetadataImpl( ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool with_tags) const { /// When the caller only needs identity (size / etag / part offsets, not tags or user metadata) we - /// take a fast path: consult the process-wide identity cache to skip the metadata request entirely - /// on repeat opens, and on a miss fetch via GetObjectAttributes (one request for size + etag + - /// multipart part sizes, with a HEAD fallback) instead of a plain HEAD. This is what removes the - /// per-file HEAD on lake scans. The tags path is unchanged. See ObjectStorageIdentityCache. + /// take a fast path: consult the identity cache to skip the metadata request entirely on repeat + /// opens, and on a miss fetch via GetObjectAttributes (one request for size + etag + multipart + /// part sizes, with a HEAD fallback) instead of a plain HEAD. This is what removes the per-file + /// HEAD on lake scans. The tags path is unchanged. See ObjectStorageIdentityCache. + /// + /// The cache is owned by the global Context and sized from server settings. Low-level + /// object-storage code can run before a global context exists; a null pointer means "no cache" + /// and the request is issued uncached. const bool identity_only = !with_tags; const String identity_key = uri.bucket + "/" + path; + ObjectStorageIdentityCachePtr identity_cache; if (identity_only) { - if (auto cached = ObjectStorageIdentityCache::instance().tryGet(identity_key)) + if (auto global_context = Context::getGlobalContextInstance()) + identity_cache = global_context->getObjectStorageIdentityCache(); + + if (identity_cache) { - ObjectMetadata result; - result.size_bytes = cached->size; - result.is_size_known = cached->is_size_known; - result.etag = cached->etag; - result.part_offsets = cached->part_offsets; - /// The cache does not store Last-Modified. Leaving it at the epoch with - /// `is_last_modified_known == true` would make schema and count caches treat the object - /// as older than every cached entry. - result.is_last_modified_known = false; - return result; + if (auto cached = identity_cache->tryGet(identity_key)) + { + ObjectMetadata result; + result.size_bytes = cached->size; + result.is_size_known = cached->is_size_known; + result.etag = cached->etag; + result.part_offsets = cached->part_offsets; + /// The cache does not store Last-Modified. Leaving it at the epoch with + /// `is_last_modified_known == true` would make schema and count caches treat the object + /// as older than every cached entry. + result.is_last_modified_known = false; + return result; + } } } @@ -1036,8 +1047,8 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool } } - if (identity_only) - ObjectStorageIdentityCache::instance().set( + if (identity_cache) + identity_cache->set( identity_key, ObjectStorageIdentity{result.etag, result.size_bytes, result.is_size_known, result.part_offsets}); diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 93f4445ea4d92..43df61fdd4337 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -46,6 +46,7 @@ #include #include #include +#include #include #include #include @@ -618,6 +619,7 @@ struct ContextSharedPart : boost::noncopyable mutable ResourceManagerPtr resource_manager; mutable UncompressedCachePtr uncompressed_cache TSA_GUARDED_BY(mutex); /// The cache of decompressed blocks. mutable MarkCachePtr mark_cache TSA_GUARDED_BY(mutex); /// Cache of marks in compressed files. + mutable ObjectStorageIdentityCachePtr object_storage_identity_cache TSA_GUARDED_BY(mutex); /// Cache of object-storage identity (size/etag/part-offsets) to avoid per-open HEADs. mutable UniqueKeyIndexCachePtr unique_key_index_cache TSA_GUARDED_BY(mutex); /// RocksDB-compatible block cache over CacheBase for the UNIQUE KEY index (nullptr when RocksDB unavailable or disabled). mutable DeleteBitmapCachePtr delete_bitmap_cache TSA_GUARDED_BY(mutex); /// UNIQUE KEY per-part delete-bitmap cache. mutable PrimaryIndexCachePtr primary_index_cache TSA_GUARDED_BY(mutex); @@ -4686,6 +4688,47 @@ void Context::clearMarkCache() const JemallocCacheArena::purge(); } +void Context::setObjectStorageIdentityCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio) +{ + std::lock_guard lock(shared->mutex); + + if (shared->object_storage_identity_cache) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Object storage identity cache has been already created."); + + /// A zero size disables the cache: leave the pointer unset so callers fall back to uncached reads. + if (max_size_in_bytes == 0) + return; + + shared->object_storage_identity_cache = std::make_shared(cache_policy, max_size_in_bytes, size_ratio); +} + +void Context::updateObjectStorageIdentityCacheConfiguration(const Poco::Util::AbstractConfiguration & config, size_t max_cache_size) +{ + std::lock_guard lock(shared->mutex); + + if (!shared->object_storage_identity_cache) + return; + + size_t size = config.getUInt64("object_storage_identity_cache_size", 67108864); + if (size > max_cache_size) + size = max_cache_size; + shared->object_storage_identity_cache->setMaxSizeInBytes(size); +} + +std::shared_ptr Context::getObjectStorageIdentityCache() const +{ + SharedLockGuard lock(shared->mutex); + return shared->object_storage_identity_cache; +} + +void Context::clearObjectStorageIdentityCache() const +{ + ObjectStorageIdentityCachePtr cache = getObjectStorageIdentityCache(); + + if (cache) + cache->clear(); +} + void Context::setUniqueKeyIndexCache( [[maybe_unused]] const String & cache_policy, [[maybe_unused]] size_t max_cache_size_in_bytes, diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 1675118688599..94f971e71b975 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -104,6 +104,7 @@ class RefreshSet; class Cluster; class Compiler; class MarkCache; +class ObjectStorageIdentityCache; class UniqueKeyIndexCache; class DeleteBitmapCache; class PrimaryIndexCache; @@ -1570,6 +1571,13 @@ class Context: public ContextData, public std::enable_shared_from_this void clearMarkCache() const; ThreadPool & getLoadMarksThreadpool() const; + /// Object-storage identity cache (per-object size/ETag/multipart-part-offsets) to avoid a HEAD + /// on every object open. A max_size_in_bytes of 0 leaves the cache unset (disabled). + void setObjectStorageIdentityCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio); + void updateObjectStorageIdentityCacheConfiguration(const Poco::Util::AbstractConfiguration & config, size_t max_cache_size); + std::shared_ptr getObjectStorageIdentityCache() const; + void clearObjectStorageIdentityCache() const; + /// UNIQUE KEY index cache: ClickHouse-side `CacheBase` adapter /// over the RocksDB block cache used by SST-backed UNIQUE KEY indexes. void setUniqueKeyIndexCache(const String & cache_policy, size_t max_cache_size_in_bytes, double size_ratio); diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 443d9d2d78a42..7889b4e7ce86b 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -515,6 +515,10 @@ BlockIO InterpreterSystemQuery::execute() #else throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "The server was compiled without the support for Parquet"); #endif + case Type::CLEAR_OBJECT_STORAGE_IDENTITY_CACHE: + getContext()->checkAccess(AccessType::SYSTEM_DROP_OBJECT_STORAGE_IDENTITY_CACHE); + system_context->clearObjectStorageIdentityCache(); + break; case Type::CLEAR_POINT_IN_POLYGON_CACHE: getContext()->checkAccess(AccessType::SYSTEM_DROP_POINT_IN_POLYGON_CACHE); clearPointInPolygonCache(); @@ -3296,6 +3300,9 @@ AccessRightsElements InterpreterSystemQuery::getRequiredAccessForDDLOnCluster() case Type::CLEAR_PARQUET_METADATA_CACHE: required_access.emplace_back(AccessType::SYSTEM_DROP_PARQUET_METADATA_CACHE); break; + case Type::CLEAR_OBJECT_STORAGE_IDENTITY_CACHE: + required_access.emplace_back(AccessType::SYSTEM_DROP_OBJECT_STORAGE_IDENTITY_CACHE); + break; case Type::CLEAR_POINT_IN_POLYGON_CACHE: required_access.emplace_back(AccessType::SYSTEM_DROP_POINT_IN_POLYGON_CACHE); break; diff --git a/src/Parsers/ASTSystemQuery.cpp b/src/Parsers/ASTSystemQuery.cpp index c5da6d8dffdfc..a3f011d06c7e0 100644 --- a/src/Parsers/ASTSystemQuery.cpp +++ b/src/Parsers/ASTSystemQuery.cpp @@ -673,6 +673,7 @@ void ASTSystemQuery::formatImpl(WriteBuffer & ostr, const FormatSettings & setti case Type::CLEAR_ICEBERG_METADATA_CACHE: case Type::CLEAR_PAIMON_METADATA_CACHE: case Type::CLEAR_PARQUET_METADATA_CACHE: + case Type::CLEAR_OBJECT_STORAGE_IDENTITY_CACHE: case Type::CLEAR_POINT_IN_POLYGON_CACHE: case Type::CLEAR_AVRO_SCHEMA_CACHE: case Type::DROP_OBJECT_STORAGE_LIST_OBJECTS_CACHE: diff --git a/src/Parsers/ASTSystemQuery.h b/src/Parsers/ASTSystemQuery.h index d6cb01ba5f909..d18eac0d789b1 100644 --- a/src/Parsers/ASTSystemQuery.h +++ b/src/Parsers/ASTSystemQuery.h @@ -49,6 +49,7 @@ class ASTSystemQuery : public IAST, public ASTQueryWithOnCluster CLEAR_ICEBERG_METADATA_CACHE, CLEAR_PAIMON_METADATA_CACHE, CLEAR_PARQUET_METADATA_CACHE, + CLEAR_OBJECT_STORAGE_IDENTITY_CACHE, CLEAR_POINT_IN_POLYGON_CACHE, CLEAR_PUFFIN_FILES_CACHE, CLEAR_FILESYSTEM_CACHE, diff --git a/src/Parsers/ParserSystemQuery.cpp b/src/Parsers/ParserSystemQuery.cpp index d750b1d94a610..adc5a59ce520e 100644 --- a/src/Parsers/ParserSystemQuery.cpp +++ b/src/Parsers/ParserSystemQuery.cpp @@ -322,6 +322,7 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & {"DROP ICEBERG METADATA CACHE", Type::CLEAR_ICEBERG_METADATA_CACHE}, {"DROP PAIMON METADATA CACHE", Type::CLEAR_PAIMON_METADATA_CACHE}, {"DROP PARQUET METADATA CACHE", Type::CLEAR_PARQUET_METADATA_CACHE}, + {"DROP OBJECT STORAGE IDENTITY CACHE", Type::CLEAR_OBJECT_STORAGE_IDENTITY_CACHE}, {"DROP POINT IN POLYGON CACHE", Type::CLEAR_POINT_IN_POLYGON_CACHE}, {"DROP PUFFIN FILES CACHE", Type::CLEAR_PUFFIN_FILES_CACHE}, {"DROP PUFFIN_FILES_CACHE", Type::CLEAR_PUFFIN_FILES_CACHE}, diff --git a/tests/queries/0_stateless/04836_system_cache_on_cluster_access_types.reference b/tests/queries/0_stateless/04836_system_cache_on_cluster_access_types.reference index 053c744b102ea..dac14b3fa0d69 100644 --- a/tests/queries/0_stateless/04836_system_cache_on_cluster_access_types.reference +++ b/tests/queries/0_stateless/04836_system_cache_on_cluster_access_types.reference @@ -8,6 +8,7 @@ SYSTEM CLEAR ICEBERG METADATA CACHE -> SYSTEM DROP ICEBERG METADATA CACHE SYSTEM CLEAR PAIMON METADATA CACHE -> SYSTEM DROP PAIMON METADATA CACHE SYSTEM CLEAR AVRO SCHEMA CACHE -> SYSTEM DROP AVRO SCHEMA CACHE SYSTEM CLEAR PARQUET METADATA CACHE -> SYSTEM DROP PARQUET METADATA CACHE +SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE -> SYSTEM DROP OBJECT STORAGE IDENTITY CACHE SYSTEM CLEAR POINT IN POLYGON CACHE -> SYSTEM DROP POINT IN POLYGON CACHE SYSTEM CLEAR PRIMARY INDEX CACHE -> SYSTEM DROP PRIMARY INDEX CACHE SYSTEM CLEAR MMAP CACHE -> SYSTEM DROP MMAP CACHE diff --git a/tests/queries/0_stateless/04836_system_cache_on_cluster_access_types.sh b/tests/queries/0_stateless/04836_system_cache_on_cluster_access_types.sh index 2c748c1570684..92b3b540d7b9f 100755 --- a/tests/queries/0_stateless/04836_system_cache_on_cluster_access_types.sh +++ b/tests/queries/0_stateless/04836_system_cache_on_cluster_access_types.sh @@ -95,6 +95,7 @@ SYSTEM CLEAR ICEBERG METADATA CACHE SYSTEM CLEAR PAIMON METADATA CACHE SYSTEM CLEAR AVRO SCHEMA CACHE SYSTEM CLEAR PARQUET METADATA CACHE +SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE SYSTEM CLEAR POINT IN POLYGON CACHE SYSTEM CLEAR PRIMARY INDEX CACHE SYSTEM CLEAR MMAP CACHE From f01689c39f79fa8e526c0f81ffb7cae4a6e8d678 Mon Sep 17 00:00:00 2001 From: UnamedRus Date: Sun, 9 Aug 2026 23:29:25 +0300 Subject: [PATCH 03/37] Update 01271_show_privileges reference for SYSTEM DROP OBJECT STORAGE IDENTITY CACHE The new `SYSTEM DROP OBJECT STORAGE IDENTITY CACHE` privilege (added with the Context-managed identity cache) adds a row to `SHOW PRIVILEGES`, so the stateless test reference needs the corresponding line. CI report: https://altinity-build-artifacts.s3.amazonaws.com/json.html?PR=2181&sha=5b16210f9e7dbf582ab7e23983337a9244f2b9fe&name_0=PR&name_1=Fast%20test Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: UnamedRus --- tests/queries/0_stateless/01271_show_privileges.reference | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/01271_show_privileges.reference b/tests/queries/0_stateless/01271_show_privileges.reference index 30e02d7c98736..9c05710825876 100644 --- a/tests/queries/0_stateless/01271_show_privileges.reference +++ b/tests/queries/0_stateless/01271_show_privileges.reference @@ -137,6 +137,7 @@ SYSTEM DROP ICEBERG METADATA CACHE ['SYSTEM CLEAR ICEBERG_METADATA_CACHE','SYSTE SYSTEM DROP PAIMON METADATA CACHE ['SYSTEM CLEAR PAIMON_METADATA_CACHE','SYSTEM DROP PAIMON_METADATA_CACHE'] GLOBAL SYSTEM DROP CACHE SYSTEM DROP AVRO SCHEMA CACHE ['SYSTEM CLEAR AVRO SCHEMA CACHE','SYSTEM DROP AVRO SCHEMA CACHE','DROP AVRO SCHEMA CACHE'] GLOBAL SYSTEM DROP CACHE SYSTEM DROP PARQUET METADATA CACHE ['SYSTEM DROP PARQUET_METADATA_CACHE'] GLOBAL SYSTEM DROP CACHE +SYSTEM DROP OBJECT STORAGE IDENTITY CACHE ['SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE','SYSTEM DROP OBJECT STORAGE IDENTITY CACHE'] GLOBAL SYSTEM DROP CACHE SYSTEM DROP POINT IN POLYGON CACHE ['SYSTEM CLEAR POINT IN POLYGON CACHE','SYSTEM DROP POINT IN POLYGON CACHE','DROP POINT IN POLYGON CACHE'] GLOBAL SYSTEM DROP CACHE SYSTEM DROP PUFFIN FILES CACHE ['SYSTEM DROP PUFFIN_FILES_CACHE'] GLOBAL SYSTEM DROP CACHE SYSTEM PREWARM PRIMARY INDEX CACHE ['SYSTEM PREWARM PRIMARY INDEX','PREWARM PRIMARY INDEX CACHE','PREWARM PRIMARY INDEX'] GLOBAL SYSTEM DROP CACHE From c653bcca1208e5e96670870e1214b82992cb322c Mon Sep 17 00:00:00 2001 From: UnamedRus Date: Thu, 13 Aug 2026 16:26:54 +0300 Subject: [PATCH 04/37] Object storage: default to HEAD for identity metadata, gate GetObjectAttributes behind a read setting Reading object metadata on a lake scan (`StorageObjectStorageSource`) needs the object size and etag (the etag keys the page / filesystem / Parquet-metadata caches). The identity fast path fetched these via `GetObjectAttributes`, which also returns the multipart part layout used for part-aligned reads - but that request is heavier than a plain `HEAD` and is issued per file, so a cold, cache-empty scan paid one `GetObjectAttributes` per object (measured ~2x slower cold on a wide-Iceberg benchmark) for part offsets it usually does not use. Default `S3ObjectStorage::getObjectMetadata` to a plain `HEAD` (size + etag, no part offsets), and add a read setting `object_storage_identity_cache_fetch_part_offsets` (default off) that opts back into `GetObjectAttributes` when part-aligned reads want the layout. A new 3-argument `IObjectStorage::getObjectMetadata` overload carries the flag; its default implementation ignores it, so non-S3 storages and the many callers that do not need part offsets are unchanged. The etag is still returned on both paths, so the object-storage caches keep working. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: UnamedRus --- src/Core/Settings.cpp | 3 +++ src/Core/SettingsChangesHistory.cpp | 1 + .../ObjectStorages/IObjectStorage.h | 10 +++++++++ .../ObjectStorages/S3/S3ObjectStorage.cpp | 21 ++++++++++++++----- .../ObjectStorages/S3/S3ObjectStorage.h | 6 +++++- src/IO/ReadSettings.h | 4 ++++ src/Interpreters/Context.cpp | 2 ++ .../StorageObjectStorageSource.cpp | 4 +++- 8 files changed, 44 insertions(+), 7 deletions(-) diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 4150173e8bb28..16e97fc4b48e5 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -7347,6 +7347,9 @@ Use userspace page cache when reading from local disks. Used for testing, unlike )", 0) \ DECLARE(Bool, use_page_cache_for_object_storage, false, R"( Use userspace page cache when reading from object storage table functions (s3, azure, hdfs) and table engines (S3, Azure, HDFS). +)", 0) \ + DECLARE(Bool, object_storage_identity_cache_fetch_part_offsets, false, R"( +Fetch the S3 multipart part layout (part offsets) when reading object metadata, using a `GetObjectAttributes` request instead of a plain `HEAD`. The part offsets enable part-aligned reads, but `GetObjectAttributes` is heavier than a `HEAD` and is issued per file, so on a cold, cache-empty scan it adds latency without benefit unless part-aligned reads are used. When disabled (the default), a `HEAD` is used, which still returns the object size and etag needed for cache keys. )", 0) \ DECLARE(Bool, read_from_page_cache_if_exists_otherwise_bypass_cache, false, R"( Use userspace page cache in passive mode, similar to read_from_filesystem_cache_if_exists_otherwise_bypass_cache. diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 1864edcad4b3e..ac666884dfc7f 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -52,6 +52,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() }); addSettingsChanges(settings_changes_history, "26.8.34.20001.altinityantalya", { + {"object_storage_identity_cache_fetch_part_offsets", false, false, "New setting: fetch the S3 multipart part layout via GetObjectAttributes when reading object metadata (for part-aligned reads). Disabled by default, in which case a cheaper HEAD (size + etag) is used, avoiding a per-file GetObjectAttributes on cold lake scans."}, {"input_format_parquet_detect_variant_by_structure", false, true, "New setting to read Spark variant columns, which have no `VARIANT` logical type, as `Dynamic` instead of `Tuple`"}, {"use_iceberg_manifest_list_partition_pruning", false, true, "New setting to skip Iceberg manifest files whose manifest-list partition summaries cannot match the query filter, without reading them."}, }); diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h b/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h index 6030296bc2792..1a15198877875 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h @@ -363,6 +363,16 @@ class IObjectStorage return getObjectMetadata(object.getPath(), with_tags); } + /// Same as above, but lets the caller ask for the object's multipart part layout (part_offsets). + /// Only S3 populates it, and only on request: it costs a heavier GetObjectAttributes call instead + /// of a plain HEAD (both return size + etag; only GetObjectAttributes returns the per-part sizes). + /// The default ignores the flag and issues the plain metadata request, so non-S3 storages and + /// callers that do not need part-aligned reads pay nothing extra. + virtual ObjectMetadata getObjectMetadata(const std::string & path, bool with_tags, bool /*fetch_part_offsets*/) const + { + return getObjectMetadata(path, with_tags); + } + /// Same as getObjectMetadata(), but ignores if object does not exist. virtual std::optional tryGetObjectMetadata(const std::string & path, bool with_tags) const = 0; virtual std::optional tryGetObjectMetadata(const RelativePathWithMetadata & object, bool with_tags) const diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp index 14529134ddc0c..6544034cf13b2 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp @@ -953,13 +953,14 @@ std::optional S3ObjectStorage::tryGetObjectMetadataImpl( return result; } -ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool with_tags) const +ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool with_tags, bool fetch_part_offsets) const { /// When the caller only needs identity (size / etag / part offsets, not tags or user metadata) we /// take a fast path: consult the identity cache to skip the metadata request entirely on repeat - /// opens, and on a miss fetch via GetObjectAttributes (one request for size + etag + multipart - /// part sizes, with a HEAD fallback) instead of a plain HEAD. This is what removes the per-file - /// HEAD on lake scans. The tags path is unchanged. See ObjectStorageIdentityCache. + /// opens. A miss uses a plain HEAD (size + etag) unless `fetch_part_offsets` is set, in which case + /// GetObjectAttributes also returns multipart part sizes and falls back to HEAD. This is what + /// removes the per-file metadata request on lake scans. The tags path is unchanged. + /// See ObjectStorageIdentityCache. /// /// The cache is owned by the global Context and sized from server settings. Low-level /// object-storage code can run before a global context exists; a null pointer means "no cache" @@ -996,7 +997,17 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool auto fetch = [&](const S3::Client & c) -> S3::ObjectInfo { if (identity_only) - return S3::getObjectIdentity(c, uri.bucket, path, /*version_id=*/ {}); + { + /// Default (fetch_part_offsets = false): a plain HEAD returns size + etag, which is all the + /// read path needs (etag keys the page / filesystem / parquet-metadata caches, size drives + /// prefetch). Only when the caller wants the multipart layout for part-aligned reads do we + /// pay the heavier GetObjectAttributes, which also returns per-part sizes; it falls back to + /// a HEAD internally if the API is unsupported or denied. This keeps a cold, cache-empty + /// lake scan on the cheap-HEAD path instead of one GetObjectAttributes per file. + if (fetch_part_offsets) + return S3::getObjectIdentity(c, uri.bucket, path, /*version_id=*/ {}); + return S3::getObjectInfo(c, uri.bucket, path, /*version_id=*/ {}, /*with_metadata=*/ false, /*with_tags=*/ false); + } return S3::getObjectInfo(c, uri.bucket, path, /*version_id=*/ {}, /*with_metadata=*/ true, /*with_tags=*/ with_tags); }; diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h index f0671586551ca..56f05448988ae 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h @@ -144,7 +144,11 @@ class S3ObjectStorage : public IObjectStorage void tagObjects(const StoredObjects & objects, const std::string & tag_key, const std::string & tag_value) override; - ObjectMetadata getObjectMetadata(const std::string & path, bool with_tags) const override; + ObjectMetadata getObjectMetadata(const std::string & path, bool with_tags) const override + { + return getObjectMetadata(path, with_tags, /*fetch_part_offsets=*/ false); + } + ObjectMetadata getObjectMetadata(const std::string & path, bool with_tags, bool fetch_part_offsets) const override; std::optional tryGetObjectMetadata(const std::string & path, bool with_tags) const override; diff --git a/src/IO/ReadSettings.h b/src/IO/ReadSettings.h index 09af30956faeb..177ed6ef35808 100644 --- a/src/IO/ReadSettings.h +++ b/src/IO/ReadSettings.h @@ -154,6 +154,10 @@ struct ReadSettings bool use_page_cache_for_object_storage = false; PageCacheSettings page_cache_settings; + /// Fetch S3 multipart part offsets (via GetObjectAttributes) when reading object metadata, for + /// part-aligned reads. Off = cheaper HEAD (size + etag only). See Settings docs. + bool object_storage_identity_cache_fetch_part_offsets = false; + /// Experimental pipeline read executor. When `enabled`, `ReadPipeline::build` sends supported reads /// through `ReaderExecutor` instead of the legacy stack of read buffers. The long-connection knobs /// apply only on the executor path. `use_long_connections` reuses one held source connection across diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 43df61fdd4337..84916c10385c3 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -391,6 +391,7 @@ namespace Setting extern const SettingsBool use_page_cache_for_disks_without_file_cache; extern const SettingsBool use_page_cache_for_local_disks; extern const SettingsBool use_page_cache_for_object_storage; + extern const SettingsBool object_storage_identity_cache_fetch_part_offsets; extern const SettingsBool use_page_cache_with_distributed_cache; extern const SettingsUInt64 use_structure_from_insertion_table_in_table_functions; extern const SettingsString workload; @@ -9037,6 +9038,7 @@ ReadSettings Context::getReadSettings() const res.use_page_cache_with_distributed_cache = settings_ref[Setting::use_page_cache_with_distributed_cache]; res.use_page_cache_for_local_disks = settings_ref[Setting::use_page_cache_for_local_disks]; res.use_page_cache_for_object_storage = settings_ref[Setting::use_page_cache_for_object_storage]; + res.object_storage_identity_cache_fetch_part_offsets = settings_ref[Setting::object_storage_identity_cache_fetch_part_offsets]; res.reader_executor.enabled = settings_ref[Setting::use_reader_executor]; res.reader_executor.use_long_connections = settings_ref[Setting::reader_executor_use_long_connections]; res.reader_executor.window_size = settings_ref[Setting::reader_executor_window_size]; diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index e73a1383c6007..aa6607ab59967 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -1973,7 +1973,9 @@ std::unique_ptr createReadBuffer( if (!object_info.metadata) { - object_info.metadata = object_storage->getObjectMetadata(object_info, /*with_tags=*/ false); + object_info.metadata = object_storage->getObjectMetadata( + object_info.getPath(), /*with_tags=*/ false, + /*fetch_part_offsets=*/ effective_read_settings.object_storage_identity_cache_fetch_part_offsets); } else if (!object_info.metadata->is_fetched && validate_etag_on_read) { From 3119a1540a835a0e19c235993dc2b6a3044d204f Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Mon, 16 Nov 2026 13:48:28 +0100 Subject: [PATCH 05/37] Object storage: initialize ObjectMetadata::part_offsets to fix build The new `ObjectMetadata::part_offsets` field has no default member initializer, so designated initializers of `ObjectMetadata` that omit it (for example in `AzureObjectStorage`) fail with `-Werror,-Wmissing-designated-field-initializers`. Give the field an explicit empty default instead of touching every call site. Co-authored-by: Cursor --- src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h b/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h index 1a15198877875..96a8357ecdc68 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h @@ -153,7 +153,7 @@ struct ObjectMetadata /// Cumulative start offsets of the object's multipart-upload parts (part i covers /// [part_offsets[i], part_offsets[i+1])). Populated best-effort via GetObjectAttributes; empty /// when unknown or single-PUT. Used to align reads to part boundaries. - std::vector part_offsets; + std::vector part_offsets = {}; /// `etag` may be used as a content-cache key (filesystem cache, page cache, Parquet /// metadata cache) only when it is present and a strong content identifier. A weak From 56df2a64e4be2e439baf39507276d420dddd8bc8 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 17 Nov 2026 13:58:54 +0100 Subject: [PATCH 06/37] Object storage: test identity cache and honor the fetch-part-offsets setting Add the stateless test `05484_object_storage_identity_cache` for the object storage identity cache. It covers a cold read that misses, a warm read that hits without `HeadObject` or `GetObjectAttributes`, the `SYSTEM DROP OBJECT STORAGE IDENTITY CACHE` and `SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE` commands, and the `object_storage_identity_cache_fetch_part_offsets` setting. The setting was only read in `createReadBuffer`, which runs after the object metadata is already filled, so it never chose `GetObjectAttributes` on the SQL read path. `KeysIterator` now receives the setting and passes it to `getObjectMetadata`, and the empty-metadata branch does the same. Co-authored-by: Cursor --- .../StorageObjectStorageSource.cpp | 19 ++- .../StorageObjectStorageSource.h | 4 +- ...84_object_storage_identity_cache.reference | 1 + .../05484_object_storage_identity_cache.sh | 108 ++++++++++++++++++ 4 files changed, 125 insertions(+), 7 deletions(-) create mode 100644 tests/queries/0_stateless/05484_object_storage_identity_cache.reference create mode 100755 tests/queries/0_stateless/05484_object_storage_identity_cache.sh diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index aa6607ab59967..95f34405399a4 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -220,6 +220,7 @@ namespace Setting extern const SettingsBool use_object_storage_list_objects_cache; extern const SettingsBool allow_experimental_iceberg_read_optimization; extern const SettingsBool azure_validate_etag_on_read; + extern const SettingsBool object_storage_identity_cache_fetch_part_offsets; } namespace @@ -458,6 +459,7 @@ std::shared_ptr StorageObjectStorageSource::createFileIterator( { const bool is_archive = configuration->isArchive(); const bool match_web_paths_only = configuration->getType() == ObjectStorageType::Web; + const bool fetch_part_offsets = local_context->getSettingsRef()[Setting::object_storage_identity_cache_fetch_part_offsets]; if (distributed_processing) { @@ -501,7 +503,7 @@ std::shared_ptr StorageObjectStorageSource::createFileIterator( iterator = std::make_unique( paths, object_storage, virtual_columns, is_archive ? nullptr : read_keys, query_settings.ignore_non_existent_file, skip_object_metadata, with_tags, - file_progress_callback); + file_progress_callback, fetch_part_offsets); } else if (reading_path.hasGlobs()) { @@ -588,7 +590,8 @@ std::shared_ptr StorageObjectStorageSource::createFileIterator( query_settings.ignore_non_existent_file, skip_object_metadata, with_tags, - file_progress_callback); + file_progress_callback, + fetch_part_offsets); } } else if (configuration->supportsFileIterator()) @@ -671,7 +674,7 @@ std::shared_ptr StorageObjectStorageSource::createFileIterator( iterator = std::make_unique( paths, object_storage, virtual_columns, is_archive ? nullptr : read_keys, query_settings.ignore_non_existent_file, /*skip_object_metadata=*/false, with_tags, - file_progress_callback); + file_progress_callback, fetch_part_offsets); } if (is_archive) @@ -1159,7 +1162,9 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade } else { - object_info->setObjectMetadata(storage_to_use->getObjectMetadata(metadata_object, with_tags)); + object_info->setObjectMetadata(storage_to_use->getObjectMetadata( + metadata_object.getPath(), with_tags, + context_->getSettingsRef()[Setting::object_storage_identity_cache_fetch_part_offsets])); } } @@ -2377,7 +2382,8 @@ StorageObjectStorageSource::KeysIterator::KeysIterator( bool ignore_non_existent_files_, bool skip_object_metadata_, bool with_tags_, - std::function file_progress_callback_) + std::function file_progress_callback_, + bool fetch_part_offsets_) : object_storage(object_storage_) , virtual_columns(virtual_columns_) , file_progress_callback(file_progress_callback_) @@ -2385,6 +2391,7 @@ StorageObjectStorageSource::KeysIterator::KeysIterator( , ignore_non_existent_files(ignore_non_existent_files_) , skip_object_metadata(skip_object_metadata_) , with_tags(with_tags_) + , fetch_part_offsets(fetch_part_offsets_) { if (read_keys_) { @@ -2418,7 +2425,7 @@ ObjectInfoPtr StorageObjectStorageSource::KeysIterator::next(size_t /* processor object_metadata = *metadata; } else - object_metadata = object_storage->getObjectMetadata(key, with_tags); + object_metadata = object_storage->getObjectMetadata(key, with_tags, fetch_part_offsets); } else { diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.h b/src/Storages/ObjectStorage/StorageObjectStorageSource.h index 196beb0e6bbf8..cacc32ac13651 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.h +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.h @@ -314,7 +314,8 @@ class StorageObjectStorageSource::KeysIterator : public IObjectIterator bool ignore_non_existent_files_, bool skip_object_metadata_, bool with_tags_, - std::function file_progress_callback = {}); + std::function file_progress_callback = {}, + bool fetch_part_offsets_ = false); ~KeysIterator() override = default; @@ -331,6 +332,7 @@ class StorageObjectStorageSource::KeysIterator : public IObjectIterator const bool ignore_non_existent_files; const bool skip_object_metadata; const bool with_tags; + const bool fetch_part_offsets; }; /* diff --git a/tests/queries/0_stateless/05484_object_storage_identity_cache.reference b/tests/queries/0_stateless/05484_object_storage_identity_cache.reference new file mode 100644 index 0000000000000..d86bac9de59ab --- /dev/null +++ b/tests/queries/0_stateless/05484_object_storage_identity_cache.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/05484_object_storage_identity_cache.sh b/tests/queries/0_stateless/05484_object_storage_identity_cache.sh new file mode 100755 index 0000000000000..a62a23b197922 --- /dev/null +++ b/tests/queries/0_stateless/05484_object_storage_identity_cache.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-parallel-replicas +# Tag no-fasttest: reads a file through the mock S3 server at localhost:11111 +# Tag no-parallel: SYSTEM DROP OBJECT STORAGE IDENTITY CACHE clears the process-global cache +# Tag no-parallel-replicas: per-query S3 request counts change when a read is spread across replicas + +# The object-storage identity cache stores size and etag (and multipart part offsets, when +# requested) so a later open of the same key does not send HEAD or GetObjectAttributes. +# A cold read misses and issues one metadata request. The next read of that key is a hit +# and issues none. SYSTEM DROP OBJECT STORAGE IDENTITY CACHE removes the entry, so the +# following read misses again. object_storage_identity_cache_fetch_part_offsets switches +# the cold request from HEAD to GetObjectAttributes; a hit still skips both. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +URL="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" +READ_SETTINGS="max_threads = 1, use_hive_partitioning = 0, parallel_replicas_for_cluster_engines = 0, enable_filesystem_cache = 0, use_page_cache_for_object_storage = 0, s3_validate_etag_on_read = 0, use_object_storage_list_objects_cache = 0" + +fail() +{ + echo "FAIL: $*" >&2 + exit 1 +} + +# $1 label, $2 extra settings (may be empty), $3 head mode (eq0|ge1|any), $4 attrs mode, $5 hits mode, $6 misses mode +read_and_check() +{ + local label=$1 + local extra=$2 + local head_mode=$3 + local attrs_mode=$4 + local hits_mode=$5 + local misses_mode=$6 + local qid="idcache_${label}_${CLICKHOUSE_DATABASE}" + local data settings row head attrs hits misses lists + + settings="${READ_SETTINGS}" + if [[ -n $extra ]]; then + settings="${settings}, ${extra}" + fi + + data=$(${CLICKHOUSE_CLIENT} --query_id "$qid" -q \ + "SELECT n FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') SETTINGS ${settings} FORMAT TabSeparated") \ + || fail "${label}: select failed" + [[ $data == "42" ]] || fail "${label}: data '${data}'" + + ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "${label}: flush logs failed" + row=$(${CLICKHOUSE_CLIENT} -q " + SELECT + ProfileEvents['S3HeadObject'], + ProfileEvents['S3GetObjectAttributes'], + ProfileEvents['ObjectStorageIdentityCacheHits'], + ProfileEvents['ObjectStorageIdentityCacheMisses'], + ProfileEvents['S3ListObjects'] + FROM system.query_log + WHERE event_date >= yesterday() + AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND query_id = '${qid}' + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC + LIMIT 1 + FORMAT TabSeparated") || fail "${label}: query_log failed" + [[ -n $row ]] || fail "${label}: no query_log row" + + IFS=$'\t' read -r head attrs hits misses lists <<< "$row" + check_count "$label" S3HeadObject "$head" "$head_mode" + check_count "$label" S3GetObjectAttributes "$attrs" "$attrs_mode" + check_count "$label" Hits "$hits" "$hits_mode" + check_count "$label" Misses "$misses" "$misses_mode" + [[ $lists == "0" ]] || fail "${label}: S3ListObjects=${lists}, expected 0 (row: ${row})" +} + +check_count() +{ + local label=$1 + local name=$2 + local got=$3 + local mode=$4 + case "$mode" in + eq0) [[ $got == "0" ]] || fail "${label}: ${name}=${got}, expected 0" ;; + ge1) [[ $got -ge 1 ]] || fail "${label}: ${name}=${got}, expected >= 1" ;; + any) ;; + *) fail "${label}: bad mode ${mode}" ;; + esac +} + +${CLICKHOUSE_CLIENT} -q " + INSERT INTO FUNCTION s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') + SELECT 42 + SETTINGS s3_truncate_on_insert = 1" || fail "insert failed" + +# Drop whatever the INSERT cached, then prove a cold read, a hit, and that DROP really clears. +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear failed" +read_and_check cold "" ge1 eq0 eq0 ge1 +read_and_check warm "" eq0 eq0 ge1 eq0 + +${CLICKHOUSE_CLIENT} -q "SYSTEM DROP OBJECT STORAGE IDENTITY CACHE" || fail "drop failed" +read_and_check after_drop "" ge1 eq0 eq0 ge1 + +# GetObjectAttributes is the cold request when part offsets are requested. A following read is a hit. +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear before attributes failed" +read_and_check attrs_cold "object_storage_identity_cache_fetch_part_offsets = 1" any ge1 eq0 ge1 +read_and_check attrs_warm "object_storage_identity_cache_fetch_part_offsets = 1" eq0 eq0 ge1 eq0 + +echo OK From b9229c4c73e36cf3a9d9a60576a6af04c7c1e4c8 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 17 Nov 2026 14:32:20 +0100 Subject: [PATCH 07/37] Object storage: key the identity cache by endpoint and region The identity cache key was the bucket and the object path. Two S3 endpoints can serve buckets with the same name, so a hit from one endpoint answered for the other with its size and ETag. Include the endpoint and the client region in the key. Fields are separated by NUL so that a slash in one field cannot make two keys equal. The new test `05485_object_storage_identity_cache_endpoint_key` reads the same object through `localhost` and `127.0.0.1` and checks that each endpoint misses on its first read and hits on its second. Co-authored-by: Cursor --- .../ObjectStorageIdentityCache.h | 40 ++++++-- .../ObjectStorages/S3/S3ObjectStorage.cpp | 4 +- ...rage_identity_cache_endpoint_key.reference | 1 + ...ect_storage_identity_cache_endpoint_key.sh | 93 +++++++++++++++++++ 4 files changed, 129 insertions(+), 9 deletions(-) create mode 100644 tests/queries/0_stateless/05485_object_storage_identity_cache_endpoint_key.reference create mode 100755 tests/queries/0_stateless/05485_object_storage_identity_cache_endpoint_key.sh diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h index 152a05c8edeb4..1e7d0aa62dd0e 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h @@ -4,6 +4,7 @@ #include #include +#include #include namespace DB @@ -13,9 +14,10 @@ namespace DB /// ETag, and (if multipart-uploaded) the byte offsets of its parts. /// /// It exists to break the ordering that forces a HEAD before every open: object caches key on -/// (path, etag), so the etag must be known up front. Here we key by path and store the etag in the -/// value, learned once (via GetObjectAttributes / a GET response) and reused. Multipart part offsets -/// ride along so reads can be aligned to part boundaries without re-probing. +/// (path, etag), so the etag must be known up front. Here the key is the store plus the path +/// (`makeObjectStorageIdentityCacheKey`) and the etag is stored in the value, learned once (via +/// GetObjectAttributes / a GET response) and reused. Multipart part offsets ride along so reads can +/// be aligned to part boundaries without re-probing. struct ObjectStorageIdentity { String etag; @@ -43,9 +45,31 @@ struct ObjectStorageIdentityWeight } }; -/// Cache of object-storage identity, keyed by object path, so opening an object does not require a -/// fresh HEAD / GetObjectAttributes on every read (a lake scan otherwise re-HEADs every candidate -/// file on every query). Backed by CacheBase (LRU/SLRU) for size-bounded eviction and thread-safety. +/// One object on one store. Bucket and path are not enough: two S3-compatible endpoints can reuse +/// the same bucket name, and an explicit region can select a different store behind one endpoint. +/// Fields are separated by NUL so a slash in the endpoint cannot alias another field. +inline String makeObjectStorageIdentityCacheKey( + std::string_view endpoint, + std::string_view region, + std::string_view bucket, + std::string_view path) +{ + String key; + key.reserve(endpoint.size() + region.size() + bucket.size() + path.size() + 3); + key.append(endpoint); + key.push_back('\0'); + key.append(region); + key.push_back('\0'); + key.append(bucket); + key.push_back('\0'); + key.append(path); + return key; +} + +/// Cache of object-storage identity, keyed by endpoint, region, bucket and object path, so opening +/// an object does not require a fresh HEAD / GetObjectAttributes on every read (a lake scan otherwise +/// re-HEADs every candidate file on every query). Backed by CacheBase (LRU/SLRU) for size-bounded +/// eviction and thread-safety. /// /// It is owned by the global Context (see Context::getObjectStorageIdentityCache), sized from server /// settings, and cleared by `SYSTEM DROP OBJECT STORAGE IDENTITY CACHE`. Low-level object-storage @@ -59,10 +83,10 @@ class ObjectStorageIdentityCache public: ObjectStorageIdentityCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio); - /// Look up an identity by path. Increments the hit/miss ProfileEvents. + /// Look up an identity by `makeObjectStorageIdentityCacheKey`. Increments the hit/miss ProfileEvents. std::optional tryGet(const String & key); - /// Insert (or overwrite) the identity for a path. + /// Insert (or overwrite) the identity for a key. void set(const String & key, ObjectStorageIdentity identity); }; diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp index 6544034cf13b2..e07432947e0cd 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp @@ -966,7 +966,9 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool /// object-storage code can run before a global context exists; a null pointer means "no cache" /// and the request is issued uncached. const bool identity_only = !with_tags; - const String identity_key = uri.bucket + "/" + path; + /// Bucket and path alone collide when two endpoints reuse a bucket name. + const String identity_key = makeObjectStorageIdentityCacheKey( + uri.endpoint, client.get()->getRegion(), uri.bucket, path); ObjectStorageIdentityCachePtr identity_cache; if (identity_only) diff --git a/tests/queries/0_stateless/05485_object_storage_identity_cache_endpoint_key.reference b/tests/queries/0_stateless/05485_object_storage_identity_cache_endpoint_key.reference new file mode 100644 index 0000000000000..d86bac9de59ab --- /dev/null +++ b/tests/queries/0_stateless/05485_object_storage_identity_cache_endpoint_key.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/05485_object_storage_identity_cache_endpoint_key.sh b/tests/queries/0_stateless/05485_object_storage_identity_cache_endpoint_key.sh new file mode 100755 index 0000000000000..6a61bed1e231f --- /dev/null +++ b/tests/queries/0_stateless/05485_object_storage_identity_cache_endpoint_key.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-parallel-replicas +# Tag no-fasttest: reads a file through the mock S3 server at localhost:11111 +# Tag no-parallel: SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE clears the process-global cache +# Tag no-parallel-replicas: per-query S3 request counts change when a read is spread across replicas + +# The identity cache key includes the endpoint. localhost and 127.0.0.1 reach the same SeaweedFS +# bucket and key, but they are different endpoint strings, so a hit on one must not answer the other. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +URL_LOCALHOST="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" +URL_LOOPBACK="http://127.0.0.1:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" +READ_SETTINGS="max_threads = 1, use_hive_partitioning = 0, parallel_replicas_for_cluster_engines = 0, enable_filesystem_cache = 0, use_page_cache_for_object_storage = 0, s3_validate_etag_on_read = 0, use_object_storage_list_objects_cache = 0" + +fail() +{ + echo "FAIL: $*" >&2 + exit 1 +} + +# $1 label, $2 url, $3 head mode (eq0|ge1|any), $4 hits mode, $5 misses mode +read_and_check() +{ + local label=$1 + local url=$2 + local head_mode=$3 + local hits_mode=$4 + local misses_mode=$5 + local qid="idcache_ep_${label}_${CLICKHOUSE_DATABASE}" + local data row head attrs hits misses lists + + data=$(${CLICKHOUSE_CLIENT} --query_id "$qid" -q \ + "SELECT n FROM s3('${url}', 'test', 'testtest', 'TSV', 'n UInt64') SETTINGS ${READ_SETTINGS} FORMAT TabSeparated") \ + || fail "${label}: select failed" + [[ $data == "42" ]] || fail "${label}: data '${data}'" + + ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "${label}: flush logs failed" + row=$(${CLICKHOUSE_CLIENT} -q " + SELECT + ProfileEvents['S3HeadObject'], + ProfileEvents['S3GetObjectAttributes'], + ProfileEvents['ObjectStorageIdentityCacheHits'], + ProfileEvents['ObjectStorageIdentityCacheMisses'], + ProfileEvents['S3ListObjects'] + FROM system.query_log + WHERE event_date >= yesterday() + AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND query_id = '${qid}' + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC + LIMIT 1 + FORMAT TabSeparated") || fail "${label}: query_log failed" + [[ -n $row ]] || fail "${label}: no query_log row" + + IFS=$'\t' read -r head attrs hits misses lists <<< "$row" + check_count "$label" S3HeadObject "$head" "$head_mode" + check_count "$label" S3GetObjectAttributes "$attrs" eq0 + check_count "$label" Hits "$hits" "$hits_mode" + check_count "$label" Misses "$misses" "$misses_mode" + [[ $lists == "0" ]] || fail "${label}: S3ListObjects=${lists}, expected 0 (row: ${row})" +} + +check_count() +{ + local label=$1 + local name=$2 + local got=$3 + local mode=$4 + case "$mode" in + eq0) [[ $got == "0" ]] || fail "${label}: ${name}=${got}, expected 0" ;; + ge1) [[ $got -ge 1 ]] || fail "${label}: ${name}=${got}, expected >= 1" ;; + any) ;; + *) fail "${label}: bad mode ${mode}" ;; + esac +} + +${CLICKHOUSE_CLIENT} -q " + INSERT INTO FUNCTION s3('${URL_LOCALHOST}', 'test', 'testtest', 'TSV', 'n UInt64') + SELECT 42 + SETTINGS s3_truncate_on_insert = 1" || fail "insert failed" + +# Drop whatever the INSERT cached. Each endpoint must miss on its own first read. +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear failed" +read_and_check localhost_cold "$URL_LOCALHOST" ge1 eq0 ge1 +read_and_check loopback_cold "$URL_LOOPBACK" ge1 eq0 ge1 +read_and_check loopback_warm "$URL_LOOPBACK" eq0 ge1 eq0 +read_and_check localhost_warm "$URL_LOCALHOST" eq0 ge1 eq0 + +echo OK From 17e10c375ac4a264ad64a072dd469a9dc601f674 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 17 Nov 2026 14:53:22 +0100 Subject: [PATCH 08/37] Object storage: drop replaced object identity from the identity cache The identity cache was keyed by endpoint, region, bucket and path, but nothing removed an entry when this server replaced or deleted the object. A later read then used the old size as the read bound and the old ETag as If-Match, so it returned truncated data or failed the read until eviction or SYSTEM DROP. - A successful single-part or multipart upload stores the new size and ETag, taken from the upload response, in the cache. If the post-upload size check fails, the entry is dropped instead. - A successful copy, a delete, a batch delete (for each key that was removed or already absent), and a conditional delete that reports a token mismatch or not-found drop the entry. - Each key has a generation number. A metadata fetch stores its result only if the generation is unchanged since before its request, so a HEAD that was in flight during a write cannot put the old identity back. - Entries expire after `object_storage_identity_cache_ttl` seconds (default 3600, zero means no expiry). This bounds the staleness when another writer replaces an object. The new test `05486_object_storage_identity_cache_overwrite` inserts a one-row object, overwrites it with three longer rows without clearing the cache, and checks that the next read is a cache hit that returns all three rows. s3_validate_etag_on_read is disabled, so a stale cached size would truncate the read. Co-authored-by: Cursor --- docs/reference/statements/system.mdx | 2 +- programs/local/LocalServer.cpp | 4 +- programs/server/Server.cpp | 4 +- src/Core/ServerSettings.cpp | 1 + .../ObjectStorageIdentityCache.cpp | 100 ++++++++++++++++-- .../ObjectStorageIdentityCache.h | 49 +++++++-- .../ObjectStorages/S3/S3ObjectStorage.cpp | 78 +++++++++++++- .../ObjectStorages/S3/S3ObjectStorage.h | 4 + src/IO/WriteBufferFromS3.cpp | 36 +++++-- src/IO/WriteBufferFromS3.h | 10 +- src/Interpreters/Context.cpp | 5 +- src/Interpreters/Context.h | 2 +- ...storage_identity_cache_overwrite.reference | 1 + ...object_storage_identity_cache_overwrite.sh | 100 ++++++++++++++++++ 14 files changed, 362 insertions(+), 34 deletions(-) create mode 100644 tests/queries/0_stateless/05486_object_storage_identity_cache_overwrite.reference create mode 100755 tests/queries/0_stateless/05486_object_storage_identity_cache_overwrite.sh diff --git a/docs/reference/statements/system.mdx b/docs/reference/statements/system.mdx index 5d05c519ef7b0..ec3b13afd120d 100644 --- a/docs/reference/statements/system.mdx +++ b/docs/reference/statements/system.mdx @@ -136,7 +136,7 @@ Clears the parquet metadata cache. ## SYSTEM CLEAR|DROP OBJECT STORAGE IDENTITY CACHE {#drop-object-storage-identity-cache} -Clears the object-storage identity cache, which stores per-object size, ETag and multipart part offsets so that opening an object does not repeat a `HEAD` or `GetObjectAttributes` request. Its size is configured with the server settings `object_storage_identity_cache_size`, `object_storage_identity_cache_policy` and `object_storage_identity_cache_size_ratio`. Set `object_storage_identity_cache_size` to `0` to disable the cache. +Clears the object-storage identity cache, which stores per-object size, ETag and multipart part offsets so that opening an object does not repeat a `HEAD` or `GetObjectAttributes` request. Its size is configured with the server settings `object_storage_identity_cache_size`, `object_storage_identity_cache_policy` and `object_storage_identity_cache_size_ratio`. Entries expire after `object_storage_identity_cache_ttl` seconds. Set `object_storage_identity_cache_size` to `0` to disable the cache. A write by this server stores the new size and ETag immediately. A copy or delete drops that object's entry immediately. ## SYSTEM CLEAR|DROP PAIMON METADATA CACHE {#drop-paimon-metadata-cache} diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index b1966840602ae..b36d3f16a5447 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -171,6 +171,7 @@ namespace ServerSetting extern const ServerSettingsDouble mark_cache_size_ratio; extern const ServerSettingsString object_storage_identity_cache_policy; extern const ServerSettingsUInt64 object_storage_identity_cache_size; + extern const ServerSettingsUInt64 object_storage_identity_cache_ttl; extern const ServerSettingsDouble object_storage_identity_cache_size_ratio; extern const ServerSettingsString unique_key_bitmap_cache_policy; extern const ServerSettingsUInt64 unique_key_bitmap_cache_size_bytes; @@ -1534,6 +1535,7 @@ void LocalServer::processConfig() String object_storage_identity_cache_policy = server_settings[ServerSetting::object_storage_identity_cache_policy]; size_t object_storage_identity_cache_size = server_settings[ServerSetting::object_storage_identity_cache_size]; + UInt64 object_storage_identity_cache_ttl = server_settings[ServerSetting::object_storage_identity_cache_ttl]; double object_storage_identity_cache_size_ratio = server_settings[ServerSetting::object_storage_identity_cache_size_ratio]; if (object_storage_identity_cache_size > max_cache_size) { @@ -1541,7 +1543,7 @@ void LocalServer::processConfig() LOG_INFO(log, "Lowered object storage identity cache size to {} because the system has limited RAM", formatReadableSizeWithBinarySuffix(object_storage_identity_cache_size)); } global_context->setObjectStorageIdentityCache( - object_storage_identity_cache_policy, object_storage_identity_cache_size, object_storage_identity_cache_size_ratio); + object_storage_identity_cache_policy, object_storage_identity_cache_size, object_storage_identity_cache_size_ratio, object_storage_identity_cache_ttl); /// UNIQUE KEY delete-bitmap cache. Zero size disables. String unique_key_bitmap_cache_policy_name = server_settings[ServerSetting::unique_key_bitmap_cache_policy]; diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 8af11d18091f1..23a9de546bee9 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -322,6 +322,7 @@ namespace ServerSetting extern const ServerSettingsDouble mark_cache_size_ratio; extern const ServerSettingsString object_storage_identity_cache_policy; extern const ServerSettingsUInt64 object_storage_identity_cache_size; + extern const ServerSettingsUInt64 object_storage_identity_cache_ttl; extern const ServerSettingsDouble object_storage_identity_cache_size_ratio; extern const ServerSettingsString unique_key_index_cache_policy; extern const ServerSettingsUInt64 unique_key_index_cache_size_bytes; @@ -2480,6 +2481,7 @@ try String object_storage_identity_cache_policy = server_settings[ServerSetting::object_storage_identity_cache_policy]; size_t object_storage_identity_cache_size = server_settings[ServerSetting::object_storage_identity_cache_size]; + UInt64 object_storage_identity_cache_ttl = server_settings[ServerSetting::object_storage_identity_cache_ttl]; double object_storage_identity_cache_size_ratio = server_settings[ServerSetting::object_storage_identity_cache_size_ratio]; if (object_storage_identity_cache_size > max_cache_size) { @@ -2487,7 +2489,7 @@ try LOG_INFO(log, "Lowered object storage identity cache size to {} because the system has limited RAM", formatReadableSizeWithBinarySuffix(object_storage_identity_cache_size)); } global_context->setObjectStorageIdentityCache( - object_storage_identity_cache_policy, object_storage_identity_cache_size, object_storage_identity_cache_size_ratio); + object_storage_identity_cache_policy, object_storage_identity_cache_size, object_storage_identity_cache_size_ratio, object_storage_identity_cache_ttl); String unique_key_index_cache_policy_name = server_settings[ServerSetting::unique_key_index_cache_policy]; size_t unique_key_index_cache_size = server_settings[ServerSetting::unique_key_index_cache_size_bytes]; diff --git a/src/Core/ServerSettings.cpp b/src/Core/ServerSettings.cpp index b8386d32b73d4..828257a370301 100644 --- a/src/Core/ServerSettings.cpp +++ b/src/Core/ServerSettings.cpp @@ -634,6 +634,7 @@ This setting can be modified at runtime and will take effect immediately. DECLARE(Double, mark_cache_prewarm_ratio, 0.95, R"(The ratio of total size of mark cache to fill during prewarm.)", 0) \ DECLARE(String, object_storage_identity_cache_policy, "SLRU", R"(Object-storage identity cache policy name (SLRU or LRU).)", 0) \ DECLARE(UInt64, object_storage_identity_cache_size, 67108864, R"(Maximum size of the object-storage identity cache in bytes. This cache stores per-object size, ETag and multipart part offsets so that opening an object does not require a fresh HEAD/GetObjectAttributes on every read. Set to 0 to disable.)", 0) \ + DECLARE(UInt64, object_storage_identity_cache_ttl, 3600, R"(Time to live of object-storage identity cache entries, in seconds. An entry older than this is a miss, and the next read requests the size and ETag again. Zero means entries live until eviction or until this server writes, copies, or deletes the object. A write by this server stores the new size and ETag immediately. A copy or delete drops the entry immediately.)", 0) \ DECLARE(Double, object_storage_identity_cache_size_ratio, 0.5, R"(The size of the protected queue (in case of SLRU policy) in the object-storage identity cache relative to the cache's total size.)", 0) \ DECLARE(String, unique_key_index_cache_policy, "SLRU", R"(UNIQUE KEY index cache policy name (SLRU or LRU).)", 0) \ DECLARE(UInt64, unique_key_index_cache_size_bytes, 1_GiB, R"(Maximum size (bytes) of the in-process cache for UNIQUE KEY index (SST) blocks. Set to 0 to disable the cache.)", 0) \ diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp index 7c6fc306ebaf7..c9adedc176ff5 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp @@ -3,6 +3,9 @@ #include #include +#include +#include + namespace ProfileEvents { extern const Event ObjectStorageIdentityCacheHits; @@ -18,7 +21,29 @@ namespace CurrentMetrics namespace DB { -ObjectStorageIdentityCache::ObjectStorageIdentityCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio) +namespace +{ + +UInt64 nowNs() +{ + return static_cast( + std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count()); +} + +bool isExpired(const ObjectStorageIdentity & identity, UInt64 ttl_seconds) +{ + constexpr UInt64 ns_per_second = 1'000'000'000ULL; + if (identity.cached_at_ns == 0) + return true; + if (ttl_seconds > std::numeric_limits::max() / ns_per_second) + return false; + return nowNs() - identity.cached_at_ns > ttl_seconds * ns_per_second; +} + +} + +ObjectStorageIdentityCache::ObjectStorageIdentityCache( + const String & cache_policy, size_t max_size_in_bytes, double size_ratio, UInt64 ttl_seconds_) : Base( cache_policy, CurrentMetrics::ObjectStorageIdentityCacheBytes, @@ -26,23 +51,84 @@ ObjectStorageIdentityCache::ObjectStorageIdentityCache(const String & cache_poli max_size_in_bytes, /*max_count*/ 0, size_ratio) + , ttl_seconds(ttl_seconds_) +{ +} + +void ObjectStorageIdentityCache::setTTLSeconds(UInt64 ttl_seconds_) { + ttl_seconds.store(ttl_seconds_, std::memory_order_relaxed); } std::optional ObjectStorageIdentityCache::tryGet(const String & key) { - if (auto mapped = Base::get(key)) + auto mapped = Base::get(key); + if (!mapped) + { + ProfileEvents::increment(ProfileEvents::ObjectStorageIdentityCacheMisses); + return std::nullopt; + } + + const UInt64 ttl = ttl_seconds.load(std::memory_order_relaxed); + if (ttl > 0 && isExpired(*mapped, ttl)) { - ProfileEvents::increment(ProfileEvents::ObjectStorageIdentityCacheHits); - return *mapped; + /// Drop only this expired resident. A write may have replaced it since `get` returned. + const UInt64 cached_at_ns = mapped->cached_at_ns; + Base::removeIfMatches(key, [cached_at_ns](const Base::MappedPtr & current) + { + return current->cached_at_ns == cached_at_ns; + }); + ProfileEvents::increment(ProfileEvents::ObjectStorageIdentityCacheMisses); + return std::nullopt; } - ProfileEvents::increment(ProfileEvents::ObjectStorageIdentityCacheMisses); - return std::nullopt; + ProfileEvents::increment(ProfileEvents::ObjectStorageIdentityCacheHits); + return *mapped; +} + +UInt64 ObjectStorageIdentityCache::epochOf(const String & key) const +{ + std::lock_guard lock(epoch_mutex); + const auto it = epoch_by_key.find(key); + if (it == epoch_by_key.end()) + return 0; + return it->second; +} + +void ObjectStorageIdentityCache::setIfEpoch(const String & key, ObjectStorageIdentity identity, UInt64 epoch) +{ + std::lock_guard lock(epoch_mutex); + const auto it = epoch_by_key.find(key); + const UInt64 current = it == epoch_by_key.end() ? 0 : it->second; + if (current != epoch) + return; + store(key, std::move(identity)); +} + +void ObjectStorageIdentityCache::replace(const String & key, ObjectStorageIdentity identity) +{ + std::lock_guard lock(epoch_mutex); + ++epoch_by_key[key]; + store(key, std::move(identity)); +} + +void ObjectStorageIdentityCache::invalidate(const String & key) +{ + std::lock_guard lock(epoch_mutex); + ++epoch_by_key[key]; + Base::remove(key); +} + +void ObjectStorageIdentityCache::clear() +{ + std::lock_guard lock(epoch_mutex); + epoch_by_key.clear(); + Base::clear(); } -void ObjectStorageIdentityCache::set(const String & key, ObjectStorageIdentity identity) +void ObjectStorageIdentityCache::store(const String & key, ObjectStorageIdentity identity) { + identity.cached_at_ns = nowNs(); Base::set(key, std::make_shared(std::move(identity))); } diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h index 1e7d0aa62dd0e..cb44b47e36841 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h @@ -2,7 +2,10 @@ #include #include +#include +#include +#include #include #include #include @@ -26,6 +29,8 @@ struct ObjectStorageIdentity /// Cumulative start offset of each multipart part (part i covers [part_offsets[i], part_offsets[i+1])). /// Empty when unknown or the object is a single PUT. std::vector part_offsets; + /// `steady_clock` nanoseconds when this entry was stored. Zero is already expired when a TTL is set. + UInt64 cached_at_ns = 0; size_t memoryUsage() const { @@ -72,22 +77,52 @@ inline String makeObjectStorageIdentityCacheKey( /// eviction and thread-safety. /// /// It is owned by the global Context (see Context::getObjectStorageIdentityCache), sized from server -/// settings, and cleared by `SYSTEM DROP OBJECT STORAGE IDENTITY CACHE`. Low-level object-storage -/// code reaches it via Context::getGlobalContextInstance() and must tolerate a null pointer (no -/// global context yet) by simply issuing the metadata request uncached. +/// settings, and cleared by `SYSTEM DROP OBJECT STORAGE IDENTITY CACHE`. Entries expire after +/// `object_storage_identity_cache_ttl` seconds (zero disables expiry). A write by this server stores +/// the new size and ETag, and a copy or delete drops the entry, so a replaced object is not read +/// with its previous size and ETag. Low-level object-storage code reaches the cache via Context::getGlobalContextInstance() +/// and must tolerate a null pointer (no global context yet) by simply issuing the metadata request +/// uncached. class ObjectStorageIdentityCache : public CacheBase, ObjectStorageIdentityWeight> { using Base = CacheBase, ObjectStorageIdentityWeight>; public: - ObjectStorageIdentityCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio); + ObjectStorageIdentityCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio, UInt64 ttl_seconds); - /// Look up an identity by `makeObjectStorageIdentityCacheKey`. Increments the hit/miss ProfileEvents. + void setTTLSeconds(UInt64 ttl_seconds); + + /// Look up an identity by `makeObjectStorageIdentityCacheKey`. An expired entry is a miss. + /// Increments the hit/miss ProfileEvents. std::optional tryGet(const String & key); - /// Insert (or overwrite) the identity for a key. - void set(const String & key, ObjectStorageIdentity identity); + /// Generation of `key`. A fetch captures this before the request and passes it to `setIfEpoch`, + /// so a write that lands during the request cannot be overwritten by that fetch's old identity. + /// Zero when the key has never been invalidated. + UInt64 epochOf(const String & key) const; + + /// Store `identity` only when `epoch` is still the generation `epochOf` returned. + void setIfEpoch(const String & key, ObjectStorageIdentity identity, UInt64 epoch); + + /// Bump the generation and store `identity`. Used after this server has replaced the object. + void replace(const String & key, ObjectStorageIdentity identity); + + /// Bump the generation and drop the entry. A fetch already in flight will not put the old identity back. + void invalidate(const String & key); + + /// Drop every entry and generation. Hides `CacheBase::clear` so generations cannot outlive the entries. + void clear(); + +private: + /// `epoch_mutex` must be held. + void store(const String & key, ObjectStorageIdentity identity); + + mutable std::mutex epoch_mutex; + /// Generations of keys this server has replaced or deleted. Absent means generation 0. + /// Guarded by `epoch_mutex`. + UnorderedMapWithMemoryTracking epoch_by_key; + std::atomic ttl_seconds; }; using ObjectStorageIdentityCachePtr = std::shared_ptr; diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp index e07432947e0cd..b26f0f10c0f26 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp @@ -467,6 +467,10 @@ std::unique_ptr S3ObjectStorage::writeObject( /// NOLIN .attempt_timeout_ms = write_settings.object_storage_attempt_timeout_ms, .connect_timeout_cap_ms = write_settings.object_storage_connect_timeout_cap_ms}); + /// Captured by value. The buffer can outlive this storage, and the cache itself is process-global. + const String identity_key = makeObjectStorageIdentityCacheKey( + uri.endpoint, used_client->getRegion(), uri.bucket, object.remote_path); + return std::make_unique( used_client, uri.bucket, @@ -476,7 +480,27 @@ std::unique_ptr S3ObjectStorage::writeObject( /// NOLIN std::move(blob_storage_log), attributes, std::move(scheduler), - disk_write_settings); + disk_write_settings, + [identity_key](size_t size, std::optional etag) + { + auto global_context = Context::getGlobalContextInstance(); + if (!global_context) + return; + auto cache = global_context->getObjectStorageIdentityCache(); + if (!cache) + return; + /// An empty ETag means the upload replaced the object but the counted size must not be cached. + if (!etag || etag->empty()) + { + cache->invalidate(identity_key); + return; + } + ObjectStorageIdentity identity; + identity.etag = std::move(*etag); + identity.size = size; + identity.is_size_known = true; + cache->replace(identity_key, std::move(identity)); + }); } @@ -571,6 +595,7 @@ void S3ObjectStorage::removeObjectImpl(const StoredObject & object, bool if_exis deleteFileFromS3(client.get(), uri.bucket, object.remote_path, if_exists, blob_storage_log, object.local_path, object.bytes_size, ProfileEvents::DiskS3DeleteObjects); + invalidateCachedIdentity(object.remote_path); } void S3ObjectStorage::removeObjectsImpl(const StoredObjects & objects, bool if_exists) @@ -600,6 +625,8 @@ void S3ObjectStorage::removeObjectsImpl(const StoredObjects & objects, bool if_e s3_capabilities, settings_ptr->request_settings[S3RequestSetting::objects_chunk_size_to_delete], blob_storage_log, local_paths_for_blob_storage_log, file_sizes_for_blob_storage_log, ProfileEvents::DiskS3DeleteObjects); + for (const auto & object : objects) + invalidateCachedIdentity(object.remote_path); } void S3ObjectStorage::removeObjectIfExists(const StoredObject & object) @@ -655,7 +682,10 @@ ConditionalRemoveResult S3ObjectStorage::removeObjectIfTokenMatchesImpl( outcome.IsSuccess() ? "" : outcome.GetError().GetMessage()); if (outcome.IsSuccess()) + { + invalidateCachedIdentity(object.remote_path); return {ConditionalRemoveOutcome::Removed, outcome.GetResult().GetDeleteMarker()}; + } const auto & err = outcome.GetError(); @@ -663,11 +693,18 @@ ConditionalRemoveResult S3ObjectStorage::removeObjectIfTokenMatchesImpl( /// (see `S3::isPreconditionFailedError` for the one policy). Callers treat 'mismatch' and 'gone' /// alike (re-validate); a genuine absence is disambiguated downstream by a HEAD re-check. if (S3::isPreconditionFailedError(err)) + { + /// The etag we held is not the object's etag, so the cached identity is not this generation. + invalidateCachedIdentity(object.remote_path); return {ConditionalRemoveOutcome::TokenMismatch, false}; + } /// The object no longer exists (404). Protocol callers treat 'mismatch' and 'gone' alike (re-validate). if (S3::isNotFoundError(err.GetErrorType())) + { + invalidateCachedIdentity(object.remote_path); return {ConditionalRemoveOutcome::NotFound, false}; + } throw S3Exception(err.GetErrorType(), "{} (Code: {}, S3 exception: '{}') while conditionally removing object with path {} from S3", @@ -699,6 +736,7 @@ void S3ObjectStorage::removeObjectsIfExistImpl( deleteFileFromS3(used_client, uri.bucket, object.remote_path, /*if_exists=*/ true, BlobStorageLogWriter::create(disk_name), object.local_path, object.bytes_size, ProfileEvents::DiskS3DeleteObjects, attempt_seed); + invalidateCachedIdentity(object.remote_path); return; } @@ -786,6 +824,21 @@ void S3ObjectStorage::removeObjectsIfExistImpl( first_error_name = err.GetCode(); } } + /// A key that was deleted, or was already gone, must not keep its previous size and ETag. + /// A key whose delete failed is still the object the cache describes. + for (const auto & object : objects) + { + bool delete_failed = false; + for (const auto & err : outcome.GetResult().GetErrors()) + { + if (err.GetKey() != object.remote_path) + continue; + if (!S3::isNotFoundError(classifyDeleteObjectsErrorCode(err.GetCode()))) + delete_failed = true; + } + if (!delete_failed) + invalidateCachedIdentity(object.remote_path); + } if (first_error_type) throw S3Exception( PreformattedMessage::create("batch removal left objects behind: [{}]", failed_keys), *first_error_type, first_error_name); @@ -994,6 +1047,11 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool } } + /// Captured after the miss and before the request. `setIfEpoch` refuses the result when a write + /// bumps the generation while this request is in flight, so the old size and ETag cannot replace + /// the identity published by that write. + const UInt64 identity_epoch = identity_cache ? identity_cache->epochOf(identity_key) : 0; + auto settings_ptr = s3_settings.get(); auto fetch = [&](const S3::Client & c) -> S3::ObjectInfo @@ -1061,9 +1119,10 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool } if (identity_cache) - identity_cache->set( + identity_cache->setIfEpoch( identity_key, - ObjectStorageIdentity{result.etag, result.size_bytes, result.is_size_known, result.part_offsets}); + ObjectStorageIdentity{result.etag, result.size_bytes, result.is_size_known, result.part_offsets}, + identity_epoch); return result; } @@ -1102,6 +1161,7 @@ void S3ObjectStorage::copyObjectToAnotherObjectStorage( // NOLINT [&, this]{ return readObject(object_from, read_settings_to_use);}, object_to_attributes, write_settings.object_storage_copy_mode); + dest_s3->invalidateCachedIdentity(object_to.remote_path); return; } catch (S3Exception & exc) @@ -1175,6 +1235,7 @@ void S3ObjectStorage::copyObject( // NOLINT [&, this]{ return readObject(object_from, read_settings_to_use);}, object_to_attributes, write_settings.object_storage_copy_mode); + invalidateCachedIdentity(object_to.remote_path); } void S3ObjectStorage::shutdown() @@ -1384,6 +1445,17 @@ bool S3ObjectStorage::hasSingleAttemptClientForTest(uint64_t request_timeout_ms, return single_attempt_clients.contains(std::make_pair(request_timeout_ms, connect_timeout_cap_ms)); } +void S3ObjectStorage::invalidateCachedIdentity(const std::string & path) const +{ + auto global_context = Context::getGlobalContextInstance(); + if (!global_context) + return; + auto cache = global_context->getObjectStorageIdentityCache(); + if (!cache) + return; + cache->invalidate(makeObjectStorageIdentityCacheKey(uri.endpoint, client.get()->getRegion(), uri.bucket, path)); +} + std::shared_ptr S3ObjectStorage::clientForRetryProfile(const ObjectStorageControlRequest & request) const { /// getSingleAttemptClient is only invoked when actually selected, so an ordinary request never diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h index 56f05448988ae..e350bfa9527b7 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h @@ -249,6 +249,10 @@ class S3ObjectStorage : public IObjectStorage std::shared_ptr clientForRetryProfile(const ObjectStorageControlRequest & request) const; + /// Drop the identity-cache entry for `path`. The generation bump stops a HEAD that started + /// against the previous object from writing that identity back. + void invalidateCachedIdentity(const std::string & path) const; + /// Runs `fn` and, if it failed because the vended credentials expired, refreshes this disk's /// client and runs it once more. `fn` must re-read the client itself, so the second run signs /// with the refreshed one. diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index ef1dc613c1549..24f8e69154bd3 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -108,7 +108,8 @@ WriteBufferFromS3::WriteBufferFromS3( BlobStorageLogWriterPtr blob_log_, std::optional object_metadata_, ThreadPoolCallbackRunnerUnsafe schedule_, - const WriteSettings & write_settings_) + const WriteSettings & write_settings_, + OnUploadComplete on_upload_complete_) : WriteBufferFromFileBase(std::min(buf_size_, static_cast(DBMS_DEFAULT_BUFFER_SIZE)), nullptr, 0) , bucket(bucket_) , key(key_) @@ -116,6 +117,7 @@ WriteBufferFromS3::WriteBufferFromS3( , write_settings(write_settings_) , client_ptr(std::move(client_ptr_)) , object_metadata(std::move(object_metadata_)) + , on_upload_complete(std::move(on_upload_complete_)) , buffer_allocation_policy(createBufferAllocationPolicy(request_settings)) , task_tracker( std::make_unique( @@ -225,17 +227,31 @@ void WriteBufferFromS3::finalizeImpl() span.addAttributeIfNotZero("clickhouse.multipart_upload_parts", multipart_tags.size()); - if (request_settings[S3RequestSetting::check_objects_after_upload]) + try { - S3::checkObjectExists(*client_ptr, bucket, key, {}, "Immediately after upload"); - - size_t actual_size = S3::getObjectSize(*client_ptr, bucket, key, {}); - if (actual_size != total_size) - throw Exception( - ErrorCodes::S3_ERROR, - "Object {} from bucket {} has unexpected size {} after upload, expected size {}, it's a bug in S3 or S3 API.", - key, bucket, actual_size, total_size); + if (request_settings[S3RequestSetting::check_objects_after_upload]) + { + S3::checkObjectExists(*client_ptr, bucket, key, {}, "Immediately after upload"); + + size_t actual_size = S3::getObjectSize(*client_ptr, bucket, key, {}); + if (actual_size != total_size) + throw Exception( + ErrorCodes::S3_ERROR, + "Object {} from bucket {} has unexpected size {} after upload, expected size {}, it's a bug in S3 or S3 API.", + key, bucket, actual_size, total_size); + } } + catch (...) + { + /// The upload already replaced the object. Tell the caller not to keep the previous identity, + /// and not to cache the size this buffer counted. + if (on_upload_complete) + on_upload_complete(0, std::nullopt); + throw; + } + + if (on_upload_complete) + on_upload_complete(total_size, object_etag); } void WriteBufferFromS3::cancelImpl() noexcept diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h index fbce538c0a431..63e5c37398ae0 100644 --- a/src/IO/WriteBufferFromS3.h +++ b/src/IO/WriteBufferFromS3.h @@ -16,7 +16,9 @@ #include #include +#include #include +#include #include #include @@ -35,6 +37,10 @@ class TaskTracker; class WriteBufferFromS3 final : public WriteBufferFromFileBase { public: + /// Called after a successful upload, including when the post-upload size check fails. + /// `etag` is empty when the uploaded bytes must not be cached (the size check disagreed). + using OnUploadComplete = std::function etag)>; + WriteBufferFromS3( std::shared_ptr client_ptr_, const String & bucket_, @@ -44,7 +50,8 @@ class WriteBufferFromS3 final : public WriteBufferFromFileBase BlobStorageLogWriterPtr blob_log_, std::optional object_metadata_ = std::nullopt, ThreadPoolCallbackRunnerUnsafe schedule_ = {}, - const WriteSettings & write_settings_ = {}); + const WriteSettings & write_settings_ = {}, + OnUploadComplete on_upload_complete_ = {}); ~WriteBufferFromS3() override; void nextImpl() override; @@ -96,6 +103,7 @@ class WriteBufferFromS3 final : public WriteBufferFromFileBase /// by getResultObjectETag() after finalize(). Written by the upload worker, read after the /// finalize barrier (happens-before), so no extra synchronization is needed. std::optional object_etag; + OnUploadComplete on_upload_complete; LoggerPtr log = getLogger("WriteBufferFromS3"); LogSeriesLimiterPtr limited_log = std::make_shared(log, 1, 5); diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 84916c10385c3..0641cf6e65e8b 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -4689,7 +4689,7 @@ void Context::clearMarkCache() const JemallocCacheArena::purge(); } -void Context::setObjectStorageIdentityCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio) +void Context::setObjectStorageIdentityCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio, UInt64 ttl_seconds) { std::lock_guard lock(shared->mutex); @@ -4700,7 +4700,7 @@ void Context::setObjectStorageIdentityCache(const String & cache_policy, size_t if (max_size_in_bytes == 0) return; - shared->object_storage_identity_cache = std::make_shared(cache_policy, max_size_in_bytes, size_ratio); + shared->object_storage_identity_cache = std::make_shared(cache_policy, max_size_in_bytes, size_ratio, ttl_seconds); } void Context::updateObjectStorageIdentityCacheConfiguration(const Poco::Util::AbstractConfiguration & config, size_t max_cache_size) @@ -4714,6 +4714,7 @@ void Context::updateObjectStorageIdentityCacheConfiguration(const Poco::Util::Ab if (size > max_cache_size) size = max_cache_size; shared->object_storage_identity_cache->setMaxSizeInBytes(size); + shared->object_storage_identity_cache->setTTLSeconds(config.getUInt64("object_storage_identity_cache_ttl", 3600)); } std::shared_ptr Context::getObjectStorageIdentityCache() const diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 94f971e71b975..6bae698406dc1 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -1573,7 +1573,7 @@ class Context: public ContextData, public std::enable_shared_from_this /// Object-storage identity cache (per-object size/ETag/multipart-part-offsets) to avoid a HEAD /// on every object open. A max_size_in_bytes of 0 leaves the cache unset (disabled). - void setObjectStorageIdentityCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio); + void setObjectStorageIdentityCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio, UInt64 ttl_seconds); void updateObjectStorageIdentityCacheConfiguration(const Poco::Util::AbstractConfiguration & config, size_t max_cache_size); std::shared_ptr getObjectStorageIdentityCache() const; void clearObjectStorageIdentityCache() const; diff --git a/tests/queries/0_stateless/05486_object_storage_identity_cache_overwrite.reference b/tests/queries/0_stateless/05486_object_storage_identity_cache_overwrite.reference new file mode 100644 index 0000000000000..d86bac9de59ab --- /dev/null +++ b/tests/queries/0_stateless/05486_object_storage_identity_cache_overwrite.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/05486_object_storage_identity_cache_overwrite.sh b/tests/queries/0_stateless/05486_object_storage_identity_cache_overwrite.sh new file mode 100755 index 0000000000000..1ddc8b37ae55d --- /dev/null +++ b/tests/queries/0_stateless/05486_object_storage_identity_cache_overwrite.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-parallel-replicas +# Tag no-fasttest: reads a file through the mock S3 server at localhost:11111 +# Tag no-parallel: the identity cache is process-global, and another test may clear it +# Tag no-parallel-replicas: per-query S3 request counts change when a read is spread across replicas + +# Overwriting an object must not leave the previous size and ETag in the identity cache. +# The upload stores the new size and ETag, so the following read is a hit and does not HEAD. +# The second insert is longer than the first. s3_validate_etag_on_read is off, so a stale +# cached size would be the read bound and the new rows would be truncated. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +URL="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" +READ_SETTINGS="max_threads = 1, use_hive_partitioning = 0, parallel_replicas_for_cluster_engines = 0, enable_filesystem_cache = 0, use_page_cache_for_object_storage = 0, s3_validate_etag_on_read = 0, use_object_storage_list_objects_cache = 0" + +fail() +{ + echo "FAIL: $*" >&2 + exit 1 +} + +# $1 label, $2 expected rows (tab-separated query result), $3 head mode, $4 hits mode, $5 misses mode +read_and_check() +{ + local label=$1 + local expected=$2 + local head_mode=$3 + local hits_mode=$4 + local misses_mode=$5 + local qid="idcache_ow_${label}_${CLICKHOUSE_DATABASE}" + local data row head attrs hits misses lists + + data=$(${CLICKHOUSE_CLIENT} --query_id "$qid" -q \ + "SELECT n FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') ORDER BY n SETTINGS ${READ_SETTINGS} FORMAT TabSeparated") \ + || fail "${label}: select failed" + [[ $data == "$expected" ]] || fail "${label}: data '${data}', expected '${expected}'" + + ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "${label}: flush logs failed" + row=$(${CLICKHOUSE_CLIENT} -q " + SELECT + ProfileEvents['S3HeadObject'], + ProfileEvents['S3GetObjectAttributes'], + ProfileEvents['ObjectStorageIdentityCacheHits'], + ProfileEvents['ObjectStorageIdentityCacheMisses'], + ProfileEvents['S3ListObjects'] + FROM system.query_log + WHERE event_date >= yesterday() + AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND query_id = '${qid}' + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC + LIMIT 1 + FORMAT TabSeparated") || fail "${label}: query_log failed" + [[ -n $row ]] || fail "${label}: no query_log row" + + IFS=$'\t' read -r head attrs hits misses lists <<< "$row" + check_count "$label" S3HeadObject "$head" "$head_mode" + check_count "$label" S3GetObjectAttributes "$attrs" eq0 + check_count "$label" Hits "$hits" "$hits_mode" + check_count "$label" Misses "$misses" "$misses_mode" + [[ $lists == "0" ]] || fail "${label}: S3ListObjects=${lists}, expected 0 (row: ${row})" +} + +check_count() +{ + local label=$1 + local name=$2 + local got=$3 + local mode=$4 + case "$mode" in + eq0) [[ $got == "0" ]] || fail "${label}: ${name}=${got}, expected 0" ;; + ge1) [[ $got -ge 1 ]] || fail "${label}: ${name}=${got}, expected >= 1" ;; + *) fail "${label}: bad mode ${mode}" ;; + esac +} + +insert_values() +{ + local values=$1 + ${CLICKHOUSE_CLIENT} -q " + INSERT INTO FUNCTION s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') + ${values} + SETTINGS s3_truncate_on_insert = 1" || fail "insert failed: ${values}" +} + +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear failed" + +insert_values "SELECT 1" +# The insert published the short object's identity, so this read does not HEAD. +read_and_check short $'1' eq0 ge1 eq0 + +# No CLEAR between the inserts. The cache is holding the short object's size until the upload replaces it. +insert_values "SELECT 10 UNION ALL SELECT 20 UNION ALL SELECT 30" +read_and_check overwritten $'10\n20\n30' eq0 ge1 eq0 + +echo OK From 5c9cd7f8fbbf28c9961b747923dff7702e73a0c8 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 17 Nov 2026 15:04:23 +0100 Subject: [PATCH 09/37] Object storage: store the quoted ETag from GetObjectAttributes `getObjectIdentity` stripped the quotes from the ETag it read with GetObjectAttributes, and its HEAD fallback did the same. `getObjectInfo` (HEAD) and PutObject keep the quotes. The identity cache stores this value and `ReadBufferFromS3` sends it as `If-Match`, so the two spellings of one object are different bytes. With `object_storage_identity_cache_fetch_part_offsets = 1` the cache held an unquoted ETag, and with `s3_validate_etag_on_read` on the next read of that object could fail although the object did not change. Both paths now store the quoted form. `quoteETag` adds the quotes when they are missing and leaves an already quoted or weak (`W/"..."`) value unchanged. The new test `05487_object_storage_identity_cache_etag_spelling` keeps `s3_validate_etag_on_read` on. It checks that the ETag read through HEAD and through GetObjectAttributes is the same quoted value, and that the following read hits the cache and returns that value. Co-authored-by: Cursor --- src/IO/S3/getObjectInfo.cpp | 20 ++-- src/IO/S3/getObjectInfo.h | 5 +- ...age_identity_cache_etag_spelling.reference | 1 + ...ct_storage_identity_cache_etag_spelling.sh | 110 ++++++++++++++++++ 4 files changed, 126 insertions(+), 10 deletions(-) create mode 100644 tests/queries/0_stateless/05487_object_storage_identity_cache_etag_spelling.reference create mode 100755 tests/queries/0_stateless/05487_object_storage_identity_cache_etag_spelling.sh diff --git a/src/IO/S3/getObjectInfo.cpp b/src/IO/S3/getObjectInfo.cpp index a4c63f5d33aa5..465cc4088da86 100644 --- a/src/IO/S3/getObjectInfo.cpp +++ b/src/IO/S3/getObjectInfo.cpp @@ -75,14 +75,18 @@ namespace return client.GetObjectAttributes(req); } - /// GetObjectAttributes returns the ETag without surrounding quotes, whereas HeadObject returns it - /// quoted. Normalize (strip quotes) so identities are consistent regardless of which path produced - /// them - important because they are used as cache keys. - String stripQuotes(String s) + /// `HeadObject` and `PutObject` spell an ETag as `"abc"` (multipart: `"abc-2"`). `GetObjectAttributes` + /// returns that value without the quotes. `If-Match` and the identity cache compare the bytes, so + /// both paths store the quoted spelling. A weak validator (`W/"abc"`) is already in HTTP form. + String quoteETag(String s) { + if (s.empty()) + return s; if (s.size() >= 2 && s.front() == '"' && s.back() == '"') - s = s.substr(1, s.size() - 2); - return s; + return s; + if (s.starts_with("W/\"") && s.ends_with('"')) + return s; + return "\"" + s + "\""; } Aws::S3::Model::GetObjectTaggingOutcome getObjectTagging( @@ -246,7 +250,7 @@ ObjectInfo getObjectIdentity( ObjectInfo object_info; object_info.size = static_cast(result.GetObjectSize()); object_info.is_size_known = true; - object_info.etag = stripQuotes(result.GetETag()); + object_info.etag = quoteETag(result.GetETag()); for (const auto & part : result.GetObjectParts().GetParts()) object_info.part_sizes.push_back(static_cast(part.GetSize())); return object_info; @@ -256,7 +260,7 @@ ObjectInfo getObjectIdentity( } ObjectInfo object_info = getObjectInfo(client, bucket, key, version_id, /*with_metadata=*/ false, /*with_tags=*/ false); - object_info.etag = stripQuotes(object_info.etag); + object_info.etag = quoteETag(object_info.etag); return object_info; } diff --git a/src/IO/S3/getObjectInfo.h b/src/IO/S3/getObjectInfo.h index 718c040c101aa..4eecfb0d101e9 100644 --- a/src/IO/S3/getObjectInfo.h +++ b/src/IO/S3/getObjectInfo.h @@ -53,8 +53,9 @@ ObjectInfo getObjectInfo( /// Fetches size + etag (+ multipart part sizes) in a single GetObjectAttributes request instead of a /// HEAD, so a caller that also wants the multipart layout gets it without a second round-trip. Falls /// back to getObjectInfo() (HEAD) if GetObjectAttributes is unsupported, denied, or otherwise errors, -/// so it is safe against S3-compatible stores that lack the API. `part_sizes` is populated only on -/// the GetObjectAttributes path for multipart objects; it is empty on the fallback path. +/// so it is safe against S3-compatible stores that lack the API. The etag is the quoted spelling +/// `HeadObject` returns, including when GetObjectAttributes omits the quotes. `part_sizes` is +/// populated only on the GetObjectAttributes path for multipart objects; it is empty on the fallback path. ObjectInfo getObjectIdentity( const S3::Client & client, const String & bucket, diff --git a/tests/queries/0_stateless/05487_object_storage_identity_cache_etag_spelling.reference b/tests/queries/0_stateless/05487_object_storage_identity_cache_etag_spelling.reference new file mode 100644 index 0000000000000..d86bac9de59ab --- /dev/null +++ b/tests/queries/0_stateless/05487_object_storage_identity_cache_etag_spelling.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/05487_object_storage_identity_cache_etag_spelling.sh b/tests/queries/0_stateless/05487_object_storage_identity_cache_etag_spelling.sh new file mode 100755 index 0000000000000..081ea29a33879 --- /dev/null +++ b/tests/queries/0_stateless/05487_object_storage_identity_cache_etag_spelling.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-parallel-replicas +# Tag no-fasttest: reads a file through the mock S3 server at localhost:11111 +# Tag no-parallel: SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE clears the process-global cache +# Tag no-parallel-replicas: per-query S3 request counts change when a read is spread across replicas + +# HeadObject spells an ETag as `"abc"`. GetObjectAttributes returns the same value without the +# quotes. The identity cache and If-Match compare those bytes, so both paths must store the quoted +# spelling. s3_validate_etag_on_read stays on: an unquoted cached ETag is sent as If-Match and the +# read fails even though the object did not change. `_etag` is the value stored for that read. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +URL="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" +READ_SETTINGS="max_threads = 1, use_hive_partitioning = 0, parallel_replicas_for_cluster_engines = 0, enable_filesystem_cache = 0, use_page_cache_for_object_storage = 0, s3_validate_etag_on_read = 1, use_object_storage_list_objects_cache = 0" + +fail() +{ + echo "FAIL: $*" >&2 + exit 1 +} + +# $1 label, $2 extra settings, $3 head mode (eq0|ge1|any), $4 attrs mode, $5 hits mode, $6 misses mode +# prints the _etag on stdout +read_etag() +{ + local label=$1 + local extra=$2 + local head_mode=$3 + local attrs_mode=$4 + local hits_mode=$5 + local misses_mode=$6 + local qid="idcache_etag_${label}_${CLICKHOUSE_DATABASE}" + local settings data row n etag head attrs hits misses lists + + settings="${READ_SETTINGS}" + if [[ -n $extra ]]; then + settings="${settings}, ${extra}" + fi + + data=$(${CLICKHOUSE_CLIENT} --query_id "$qid" -q \ + "SELECT n, _etag FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') SETTINGS ${settings} FORMAT TabSeparated") \ + || fail "${label}: select failed" + IFS=$'\t' read -r n etag <<< "$data" + [[ $n == "42" ]] || fail "${label}: data '${data}'" + [[ $etag == '"'*'"' ]] || fail "${label}: etag '${etag}' is not quoted" + + ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "${label}: flush logs failed" + row=$(${CLICKHOUSE_CLIENT} -q " + SELECT + ProfileEvents['S3HeadObject'], + ProfileEvents['S3GetObjectAttributes'], + ProfileEvents['ObjectStorageIdentityCacheHits'], + ProfileEvents['ObjectStorageIdentityCacheMisses'], + ProfileEvents['S3ListObjects'] + FROM system.query_log + WHERE event_date >= yesterday() + AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND query_id = '${qid}' + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC + LIMIT 1 + FORMAT TabSeparated") || fail "${label}: query_log failed" + [[ -n $row ]] || fail "${label}: no query_log row" + + IFS=$'\t' read -r head attrs hits misses lists <<< "$row" + check_count "$label" S3HeadObject "$head" "$head_mode" + check_count "$label" S3GetObjectAttributes "$attrs" "$attrs_mode" + check_count "$label" Hits "$hits" "$hits_mode" + check_count "$label" Misses "$misses" "$misses_mode" + [[ $lists == "0" ]] || fail "${label}: S3ListObjects=${lists}, expected 0 (row: ${row})" + + printf '%s' "$etag" +} + +check_count() +{ + local label=$1 + local name=$2 + local got=$3 + local mode=$4 + case "$mode" in + eq0) [[ $got == "0" ]] || fail "${label}: ${name}=${got}, expected 0" ;; + ge1) [[ $got -ge 1 ]] || fail "${label}: ${name}=${got}, expected >= 1" ;; + any) ;; + *) fail "${label}: bad mode ${mode}" ;; + esac +} + +${CLICKHOUSE_CLIENT} -q " + INSERT INTO FUNCTION s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') + SELECT 42 + SETTINGS s3_truncate_on_insert = 1" || fail "insert failed" + +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear failed" +head_etag=$(read_etag head "" ge1 eq0 eq0 ge1) || exit 1 + +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear before attributes failed" +# GetObjectAttributes may fall back to HEAD, so the HEAD counter is unconstrained. +attrs_etag=$(read_etag attrs "object_storage_identity_cache_fetch_part_offsets = 1" any ge1 eq0 ge1) || exit 1 +[[ $attrs_etag == "$head_etag" ]] || fail "etag spelling differs: head '${head_etag}', attributes '${attrs_etag}'" + +# The attributes read filled the cache. The next read must reuse that ETag, not HEAD again. +hit_etag=$(read_etag attrs_hit "object_storage_identity_cache_fetch_part_offsets = 1" eq0 eq0 ge1 eq0) || exit 1 +[[ $hit_etag == "$head_etag" ]] || fail "cached etag '${hit_etag}' differs from head '${head_etag}'" + +echo OK From 91e2cf38db8e7c9999986317abe8f8c1830bcb14 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 17 Nov 2026 15:12:14 +0100 Subject: [PATCH 10/37] Object storage: create the identity cache on config reload `Context::updateObjectStorageIdentityCacheConfiguration` returned at once when the identity cache pointer was unset. Startup leaves the pointer unset when `object_storage_identity_cache_size` is `0`, so a reload that set a non-zero size did nothing and every read stayed uncached until the next process start. A reload now creates the cache when the new size is non-zero, using the reloaded policy, size ratio, and TTL. The size is still capped by the same RAM limit as at startup. A reload that keeps the size at `0` still leaves the cache disabled. The setting description and the `SYSTEM CLEAR|DROP OBJECT STORAGE IDENTITY CACHE` page say that `SYSTEM RELOAD CONFIG` turns the cache on. Co-authored-by: Cursor --- docs/reference/statements/system.mdx | 2 +- src/Core/ServerSettings.cpp | 2 +- src/Interpreters/Context.cpp | 21 +++++++++++++++++---- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/docs/reference/statements/system.mdx b/docs/reference/statements/system.mdx index ec3b13afd120d..7cbc84ba5d14a 100644 --- a/docs/reference/statements/system.mdx +++ b/docs/reference/statements/system.mdx @@ -136,7 +136,7 @@ Clears the parquet metadata cache. ## SYSTEM CLEAR|DROP OBJECT STORAGE IDENTITY CACHE {#drop-object-storage-identity-cache} -Clears the object-storage identity cache, which stores per-object size, ETag and multipart part offsets so that opening an object does not repeat a `HEAD` or `GetObjectAttributes` request. Its size is configured with the server settings `object_storage_identity_cache_size`, `object_storage_identity_cache_policy` and `object_storage_identity_cache_size_ratio`. Entries expire after `object_storage_identity_cache_ttl` seconds. Set `object_storage_identity_cache_size` to `0` to disable the cache. A write by this server stores the new size and ETag immediately. A copy or delete drops that object's entry immediately. +Clears the object-storage identity cache, which stores per-object size, ETag and multipart part offsets so that opening an object does not repeat a `HEAD` or `GetObjectAttributes` request. Its size is configured with the server settings `object_storage_identity_cache_size`, `object_storage_identity_cache_policy` and `object_storage_identity_cache_size_ratio`. Entries expire after `object_storage_identity_cache_ttl` seconds. Set `object_storage_identity_cache_size` to `0` to disable the cache. `SYSTEM RELOAD CONFIG` with a non-zero size creates it when startup left it disabled. A write by this server stores the new size and ETag immediately. A copy or delete drops that object's entry immediately. ## SYSTEM CLEAR|DROP PAIMON METADATA CACHE {#drop-paimon-metadata-cache} diff --git a/src/Core/ServerSettings.cpp b/src/Core/ServerSettings.cpp index 828257a370301..2bbf14a520483 100644 --- a/src/Core/ServerSettings.cpp +++ b/src/Core/ServerSettings.cpp @@ -633,7 +633,7 @@ This setting can be modified at runtime and will take effect immediately. DECLARE(Double, mark_cache_size_ratio, DEFAULT_MARK_CACHE_SIZE_RATIO, R"(The size of the protected queue (in case of SLRU policy) in the mark cache relative to the cache's total size.)", 0) \ DECLARE(Double, mark_cache_prewarm_ratio, 0.95, R"(The ratio of total size of mark cache to fill during prewarm.)", 0) \ DECLARE(String, object_storage_identity_cache_policy, "SLRU", R"(Object-storage identity cache policy name (SLRU or LRU).)", 0) \ - DECLARE(UInt64, object_storage_identity_cache_size, 67108864, R"(Maximum size of the object-storage identity cache in bytes. This cache stores per-object size, ETag and multipart part offsets so that opening an object does not require a fresh HEAD/GetObjectAttributes on every read. Set to 0 to disable.)", 0) \ + DECLARE(UInt64, object_storage_identity_cache_size, 67108864, R"(Maximum size of the object-storage identity cache in bytes. This cache stores per-object size, ETag and multipart part offsets so that opening an object does not require a fresh HEAD/GetObjectAttributes on every read. Set to 0 to disable. A reload (`SYSTEM RELOAD CONFIG`) with a non-zero size creates the cache when startup left it disabled, so a restart is not required to turn it on.)", 0) \ DECLARE(UInt64, object_storage_identity_cache_ttl, 3600, R"(Time to live of object-storage identity cache entries, in seconds. An entry older than this is a miss, and the next read requests the size and ETag again. Zero means entries live until eviction or until this server writes, copies, or deletes the object. A write by this server stores the new size and ETag immediately. A copy or delete drops the entry immediately.)", 0) \ DECLARE(Double, object_storage_identity_cache_size_ratio, 0.5, R"(The size of the protected queue (in case of SLRU policy) in the object-storage identity cache relative to the cache's total size.)", 0) \ DECLARE(String, unique_key_index_cache_policy, "SLRU", R"(UNIQUE KEY index cache policy name (SLRU or LRU).)", 0) \ diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 0641cf6e65e8b..6845c38f1bfca 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -4697,6 +4697,7 @@ void Context::setObjectStorageIdentityCache(const String & cache_policy, size_t throw Exception(ErrorCodes::LOGICAL_ERROR, "Object storage identity cache has been already created."); /// A zero size disables the cache: leave the pointer unset so callers fall back to uncached reads. + /// `updateObjectStorageIdentityCacheConfiguration` creates it later if a reload sets a non-zero size. if (max_size_in_bytes == 0) return; @@ -4707,14 +4708,26 @@ void Context::updateObjectStorageIdentityCacheConfiguration(const Poco::Util::Ab { std::lock_guard lock(shared->mutex); - if (!shared->object_storage_identity_cache) - return; - size_t size = config.getUInt64("object_storage_identity_cache_size", 67108864); if (size > max_cache_size) size = max_cache_size; + const auto ttl_seconds = config.getUInt64("object_storage_identity_cache_ttl", 3600); + + /// Startup with size 0 leaves the pointer unset. A reload that raises the size has to create the + /// cache here: returning would keep every read uncached until the next process start. + if (!shared->object_storage_identity_cache) + { + if (size == 0) + return; + + const String policy = config.getString("object_storage_identity_cache_policy", "SLRU"); + const double size_ratio = config.getDouble("object_storage_identity_cache_size_ratio", 0.5); + shared->object_storage_identity_cache = std::make_shared(policy, size, size_ratio, ttl_seconds); + return; + } + shared->object_storage_identity_cache->setMaxSizeInBytes(size); - shared->object_storage_identity_cache->setTTLSeconds(config.getUInt64("object_storage_identity_cache_ttl", 3600)); + shared->object_storage_identity_cache->setTTLSeconds(ttl_seconds); } std::shared_ptr Context::getObjectStorageIdentityCache() const From 8421641eca7ea709fcee58d9941fe24f72ceb3fe Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 17 Nov 2026 15:49:30 +0100 Subject: [PATCH 11/37] Object storage: simplify the identity cache The overwrite fix added a per-key generation map, a TTL with a timestamp on every entry, and a region in the cache key. None of these is needed for the overwrite case: a write, copy, or delete by this server removes the entry, and the next read requests the metadata again. Keeping them made the cache, the S3 write path, and the settings larger than the fix requires. - ObjectStorageIdentityCache is a plain CacheBase again: tryGet and set. The generation map, setIfEpoch, epochOf, replace, invalidate, and the TTL with its timestamp are removed. - The cache key is the endpoint, the bucket, and the path. The region is removed. - A successful upload calls the upload callback without arguments and the callback removes the entry. The size and ETag are no longer copied from the upload response. - A copy, a delete, a batch delete, and a conditional delete remove the entry through invalidateCachedIdentity. - The object_storage_identity_cache_ttl server setting is removed. - Context::setObjectStorageIdentityCache no longer takes a TTL. The reload path that creates the cache when startup left it disabled, and the ETag spelling fix, are unchanged. The test 05486 expects the read after an overwrite to miss and return all three new rows, and the next read to hit. Co-authored-by: Cursor --- docs/reference/statements/system.mdx | 2 +- programs/local/LocalServer.cpp | 4 +- programs/server/Server.cpp | 4 +- src/Core/ServerSettings.cpp | 1 - .../ObjectStorageIdentityCache.cpp | 100 ++---------------- .../ObjectStorageIdentityCache.h | 71 +++---------- .../ObjectStorages/S3/S3ObjectStorage.cpp | 40 ++----- .../ObjectStorages/S3/S3ObjectStorage.h | 3 +- src/IO/WriteBufferFromS3.cpp | 7 +- src/IO/WriteBufferFromS3.h | 5 +- src/Interpreters/Context.cpp | 8 +- src/Interpreters/Context.h | 2 +- ...object_storage_identity_cache_overwrite.sh | 16 +-- 13 files changed, 52 insertions(+), 211 deletions(-) diff --git a/docs/reference/statements/system.mdx b/docs/reference/statements/system.mdx index 7cbc84ba5d14a..fc8e43c893b21 100644 --- a/docs/reference/statements/system.mdx +++ b/docs/reference/statements/system.mdx @@ -136,7 +136,7 @@ Clears the parquet metadata cache. ## SYSTEM CLEAR|DROP OBJECT STORAGE IDENTITY CACHE {#drop-object-storage-identity-cache} -Clears the object-storage identity cache, which stores per-object size, ETag and multipart part offsets so that opening an object does not repeat a `HEAD` or `GetObjectAttributes` request. Its size is configured with the server settings `object_storage_identity_cache_size`, `object_storage_identity_cache_policy` and `object_storage_identity_cache_size_ratio`. Entries expire after `object_storage_identity_cache_ttl` seconds. Set `object_storage_identity_cache_size` to `0` to disable the cache. `SYSTEM RELOAD CONFIG` with a non-zero size creates it when startup left it disabled. A write by this server stores the new size and ETag immediately. A copy or delete drops that object's entry immediately. +Clears the object-storage identity cache, which stores per-object size, ETag and multipart part offsets so that opening an object does not repeat a `HEAD` or `GetObjectAttributes` request. Its size is configured with the server settings `object_storage_identity_cache_size`, `object_storage_identity_cache_policy` and `object_storage_identity_cache_size_ratio`. Set `object_storage_identity_cache_size` to `0` to disable the cache. `SYSTEM RELOAD CONFIG` with a non-zero size creates it when startup left it disabled. A write, copy, or delete by this server drops that object's entry, and the next read requests the metadata again. ## SYSTEM CLEAR|DROP PAIMON METADATA CACHE {#drop-paimon-metadata-cache} diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index b36d3f16a5447..b1966840602ae 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -171,7 +171,6 @@ namespace ServerSetting extern const ServerSettingsDouble mark_cache_size_ratio; extern const ServerSettingsString object_storage_identity_cache_policy; extern const ServerSettingsUInt64 object_storage_identity_cache_size; - extern const ServerSettingsUInt64 object_storage_identity_cache_ttl; extern const ServerSettingsDouble object_storage_identity_cache_size_ratio; extern const ServerSettingsString unique_key_bitmap_cache_policy; extern const ServerSettingsUInt64 unique_key_bitmap_cache_size_bytes; @@ -1535,7 +1534,6 @@ void LocalServer::processConfig() String object_storage_identity_cache_policy = server_settings[ServerSetting::object_storage_identity_cache_policy]; size_t object_storage_identity_cache_size = server_settings[ServerSetting::object_storage_identity_cache_size]; - UInt64 object_storage_identity_cache_ttl = server_settings[ServerSetting::object_storage_identity_cache_ttl]; double object_storage_identity_cache_size_ratio = server_settings[ServerSetting::object_storage_identity_cache_size_ratio]; if (object_storage_identity_cache_size > max_cache_size) { @@ -1543,7 +1541,7 @@ void LocalServer::processConfig() LOG_INFO(log, "Lowered object storage identity cache size to {} because the system has limited RAM", formatReadableSizeWithBinarySuffix(object_storage_identity_cache_size)); } global_context->setObjectStorageIdentityCache( - object_storage_identity_cache_policy, object_storage_identity_cache_size, object_storage_identity_cache_size_ratio, object_storage_identity_cache_ttl); + object_storage_identity_cache_policy, object_storage_identity_cache_size, object_storage_identity_cache_size_ratio); /// UNIQUE KEY delete-bitmap cache. Zero size disables. String unique_key_bitmap_cache_policy_name = server_settings[ServerSetting::unique_key_bitmap_cache_policy]; diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 23a9de546bee9..8af11d18091f1 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -322,7 +322,6 @@ namespace ServerSetting extern const ServerSettingsDouble mark_cache_size_ratio; extern const ServerSettingsString object_storage_identity_cache_policy; extern const ServerSettingsUInt64 object_storage_identity_cache_size; - extern const ServerSettingsUInt64 object_storage_identity_cache_ttl; extern const ServerSettingsDouble object_storage_identity_cache_size_ratio; extern const ServerSettingsString unique_key_index_cache_policy; extern const ServerSettingsUInt64 unique_key_index_cache_size_bytes; @@ -2481,7 +2480,6 @@ try String object_storage_identity_cache_policy = server_settings[ServerSetting::object_storage_identity_cache_policy]; size_t object_storage_identity_cache_size = server_settings[ServerSetting::object_storage_identity_cache_size]; - UInt64 object_storage_identity_cache_ttl = server_settings[ServerSetting::object_storage_identity_cache_ttl]; double object_storage_identity_cache_size_ratio = server_settings[ServerSetting::object_storage_identity_cache_size_ratio]; if (object_storage_identity_cache_size > max_cache_size) { @@ -2489,7 +2487,7 @@ try LOG_INFO(log, "Lowered object storage identity cache size to {} because the system has limited RAM", formatReadableSizeWithBinarySuffix(object_storage_identity_cache_size)); } global_context->setObjectStorageIdentityCache( - object_storage_identity_cache_policy, object_storage_identity_cache_size, object_storage_identity_cache_size_ratio, object_storage_identity_cache_ttl); + object_storage_identity_cache_policy, object_storage_identity_cache_size, object_storage_identity_cache_size_ratio); String unique_key_index_cache_policy_name = server_settings[ServerSetting::unique_key_index_cache_policy]; size_t unique_key_index_cache_size = server_settings[ServerSetting::unique_key_index_cache_size_bytes]; diff --git a/src/Core/ServerSettings.cpp b/src/Core/ServerSettings.cpp index 2bbf14a520483..aa18e5b4c8117 100644 --- a/src/Core/ServerSettings.cpp +++ b/src/Core/ServerSettings.cpp @@ -634,7 +634,6 @@ This setting can be modified at runtime and will take effect immediately. DECLARE(Double, mark_cache_prewarm_ratio, 0.95, R"(The ratio of total size of mark cache to fill during prewarm.)", 0) \ DECLARE(String, object_storage_identity_cache_policy, "SLRU", R"(Object-storage identity cache policy name (SLRU or LRU).)", 0) \ DECLARE(UInt64, object_storage_identity_cache_size, 67108864, R"(Maximum size of the object-storage identity cache in bytes. This cache stores per-object size, ETag and multipart part offsets so that opening an object does not require a fresh HEAD/GetObjectAttributes on every read. Set to 0 to disable. A reload (`SYSTEM RELOAD CONFIG`) with a non-zero size creates the cache when startup left it disabled, so a restart is not required to turn it on.)", 0) \ - DECLARE(UInt64, object_storage_identity_cache_ttl, 3600, R"(Time to live of object-storage identity cache entries, in seconds. An entry older than this is a miss, and the next read requests the size and ETag again. Zero means entries live until eviction or until this server writes, copies, or deletes the object. A write by this server stores the new size and ETag immediately. A copy or delete drops the entry immediately.)", 0) \ DECLARE(Double, object_storage_identity_cache_size_ratio, 0.5, R"(The size of the protected queue (in case of SLRU policy) in the object-storage identity cache relative to the cache's total size.)", 0) \ DECLARE(String, unique_key_index_cache_policy, "SLRU", R"(UNIQUE KEY index cache policy name (SLRU or LRU).)", 0) \ DECLARE(UInt64, unique_key_index_cache_size_bytes, 1_GiB, R"(Maximum size (bytes) of the in-process cache for UNIQUE KEY index (SST) blocks. Set to 0 to disable the cache.)", 0) \ diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp index c9adedc176ff5..7c6fc306ebaf7 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp @@ -3,9 +3,6 @@ #include #include -#include -#include - namespace ProfileEvents { extern const Event ObjectStorageIdentityCacheHits; @@ -21,29 +18,7 @@ namespace CurrentMetrics namespace DB { -namespace -{ - -UInt64 nowNs() -{ - return static_cast( - std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count()); -} - -bool isExpired(const ObjectStorageIdentity & identity, UInt64 ttl_seconds) -{ - constexpr UInt64 ns_per_second = 1'000'000'000ULL; - if (identity.cached_at_ns == 0) - return true; - if (ttl_seconds > std::numeric_limits::max() / ns_per_second) - return false; - return nowNs() - identity.cached_at_ns > ttl_seconds * ns_per_second; -} - -} - -ObjectStorageIdentityCache::ObjectStorageIdentityCache( - const String & cache_policy, size_t max_size_in_bytes, double size_ratio, UInt64 ttl_seconds_) +ObjectStorageIdentityCache::ObjectStorageIdentityCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio) : Base( cache_policy, CurrentMetrics::ObjectStorageIdentityCacheBytes, @@ -51,84 +26,23 @@ ObjectStorageIdentityCache::ObjectStorageIdentityCache( max_size_in_bytes, /*max_count*/ 0, size_ratio) - , ttl_seconds(ttl_seconds_) -{ -} - -void ObjectStorageIdentityCache::setTTLSeconds(UInt64 ttl_seconds_) { - ttl_seconds.store(ttl_seconds_, std::memory_order_relaxed); } std::optional ObjectStorageIdentityCache::tryGet(const String & key) { - auto mapped = Base::get(key); - if (!mapped) - { - ProfileEvents::increment(ProfileEvents::ObjectStorageIdentityCacheMisses); - return std::nullopt; - } - - const UInt64 ttl = ttl_seconds.load(std::memory_order_relaxed); - if (ttl > 0 && isExpired(*mapped, ttl)) + if (auto mapped = Base::get(key)) { - /// Drop only this expired resident. A write may have replaced it since `get` returned. - const UInt64 cached_at_ns = mapped->cached_at_ns; - Base::removeIfMatches(key, [cached_at_ns](const Base::MappedPtr & current) - { - return current->cached_at_ns == cached_at_ns; - }); - ProfileEvents::increment(ProfileEvents::ObjectStorageIdentityCacheMisses); - return std::nullopt; + ProfileEvents::increment(ProfileEvents::ObjectStorageIdentityCacheHits); + return *mapped; } - ProfileEvents::increment(ProfileEvents::ObjectStorageIdentityCacheHits); - return *mapped; -} - -UInt64 ObjectStorageIdentityCache::epochOf(const String & key) const -{ - std::lock_guard lock(epoch_mutex); - const auto it = epoch_by_key.find(key); - if (it == epoch_by_key.end()) - return 0; - return it->second; -} - -void ObjectStorageIdentityCache::setIfEpoch(const String & key, ObjectStorageIdentity identity, UInt64 epoch) -{ - std::lock_guard lock(epoch_mutex); - const auto it = epoch_by_key.find(key); - const UInt64 current = it == epoch_by_key.end() ? 0 : it->second; - if (current != epoch) - return; - store(key, std::move(identity)); -} - -void ObjectStorageIdentityCache::replace(const String & key, ObjectStorageIdentity identity) -{ - std::lock_guard lock(epoch_mutex); - ++epoch_by_key[key]; - store(key, std::move(identity)); -} - -void ObjectStorageIdentityCache::invalidate(const String & key) -{ - std::lock_guard lock(epoch_mutex); - ++epoch_by_key[key]; - Base::remove(key); -} - -void ObjectStorageIdentityCache::clear() -{ - std::lock_guard lock(epoch_mutex); - epoch_by_key.clear(); - Base::clear(); + ProfileEvents::increment(ProfileEvents::ObjectStorageIdentityCacheMisses); + return std::nullopt; } -void ObjectStorageIdentityCache::store(const String & key, ObjectStorageIdentity identity) +void ObjectStorageIdentityCache::set(const String & key, ObjectStorageIdentity identity) { - identity.cached_at_ns = nowNs(); Base::set(key, std::make_shared(std::move(identity))); } diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h index cb44b47e36841..079015a3d0ca8 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h @@ -2,10 +2,7 @@ #include #include -#include -#include -#include #include #include #include @@ -29,8 +26,6 @@ struct ObjectStorageIdentity /// Cumulative start offset of each multipart part (part i covers [part_offsets[i], part_offsets[i+1])). /// Empty when unknown or the object is a single PUT. std::vector part_offsets; - /// `steady_clock` nanoseconds when this entry was stored. Zero is already expired when a TTL is set. - UInt64 cached_at_ns = 0; size_t memoryUsage() const { @@ -51,78 +46,42 @@ struct ObjectStorageIdentityWeight }; /// One object on one store. Bucket and path are not enough: two S3-compatible endpoints can reuse -/// the same bucket name, and an explicit region can select a different store behind one endpoint. -/// Fields are separated by NUL so a slash in the endpoint cannot alias another field. -inline String makeObjectStorageIdentityCacheKey( - std::string_view endpoint, - std::string_view region, - std::string_view bucket, - std::string_view path) +/// the same bucket name. Fields are separated by NUL so a slash in the endpoint cannot alias another field. +inline String makeObjectStorageIdentityCacheKey(std::string_view endpoint, std::string_view bucket, std::string_view path) { String key; - key.reserve(endpoint.size() + region.size() + bucket.size() + path.size() + 3); + key.reserve(endpoint.size() + bucket.size() + path.size() + 2); key.append(endpoint); key.push_back('\0'); - key.append(region); - key.push_back('\0'); key.append(bucket); key.push_back('\0'); key.append(path); return key; } -/// Cache of object-storage identity, keyed by endpoint, region, bucket and object path, so opening -/// an object does not require a fresh HEAD / GetObjectAttributes on every read (a lake scan otherwise -/// re-HEADs every candidate file on every query). Backed by CacheBase (LRU/SLRU) for size-bounded -/// eviction and thread-safety. +/// Cache of object-storage identity, keyed by endpoint, bucket and object path, so opening an object +/// does not require a fresh HEAD / GetObjectAttributes on every read (a lake scan otherwise re-HEADs +/// every candidate file on every query). Backed by CacheBase (LRU/SLRU) for size-bounded eviction +/// and thread-safety. /// /// It is owned by the global Context (see Context::getObjectStorageIdentityCache), sized from server -/// settings, and cleared by `SYSTEM DROP OBJECT STORAGE IDENTITY CACHE`. Entries expire after -/// `object_storage_identity_cache_ttl` seconds (zero disables expiry). A write by this server stores -/// the new size and ETag, and a copy or delete drops the entry, so a replaced object is not read -/// with its previous size and ETag. Low-level object-storage code reaches the cache via Context::getGlobalContextInstance() -/// and must tolerate a null pointer (no global context yet) by simply issuing the metadata request -/// uncached. +/// settings, and cleared by `SYSTEM DROP OBJECT STORAGE IDENTITY CACHE`. A write, copy, or delete by +/// this server removes the entry; the next read of that key requests the metadata again. Low-level +/// object-storage code reaches the cache via Context::getGlobalContextInstance() and must tolerate a +/// null pointer (no global context yet) by simply issuing the metadata request uncached. class ObjectStorageIdentityCache : public CacheBase, ObjectStorageIdentityWeight> { using Base = CacheBase, ObjectStorageIdentityWeight>; public: - ObjectStorageIdentityCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio, UInt64 ttl_seconds); - - void setTTLSeconds(UInt64 ttl_seconds); + ObjectStorageIdentityCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio); - /// Look up an identity by `makeObjectStorageIdentityCacheKey`. An expired entry is a miss. - /// Increments the hit/miss ProfileEvents. + /// Look up an identity by `makeObjectStorageIdentityCacheKey`. Increments the hit/miss ProfileEvents. std::optional tryGet(const String & key); - /// Generation of `key`. A fetch captures this before the request and passes it to `setIfEpoch`, - /// so a write that lands during the request cannot be overwritten by that fetch's old identity. - /// Zero when the key has never been invalidated. - UInt64 epochOf(const String & key) const; - - /// Store `identity` only when `epoch` is still the generation `epochOf` returned. - void setIfEpoch(const String & key, ObjectStorageIdentity identity, UInt64 epoch); - - /// Bump the generation and store `identity`. Used after this server has replaced the object. - void replace(const String & key, ObjectStorageIdentity identity); - - /// Bump the generation and drop the entry. A fetch already in flight will not put the old identity back. - void invalidate(const String & key); - - /// Drop every entry and generation. Hides `CacheBase::clear` so generations cannot outlive the entries. - void clear(); - -private: - /// `epoch_mutex` must be held. - void store(const String & key, ObjectStorageIdentity identity); - - mutable std::mutex epoch_mutex; - /// Generations of keys this server has replaced or deleted. Absent means generation 0. - /// Guarded by `epoch_mutex`. - UnorderedMapWithMemoryTracking epoch_by_key; - std::atomic ttl_seconds; + /// Insert (or overwrite) the identity for a key. + void set(const String & key, ObjectStorageIdentity identity); }; using ObjectStorageIdentityCachePtr = std::shared_ptr; diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp index b26f0f10c0f26..21bff1cdcb61a 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp @@ -468,8 +468,7 @@ std::unique_ptr S3ObjectStorage::writeObject( /// NOLIN .connect_timeout_cap_ms = write_settings.object_storage_connect_timeout_cap_ms}); /// Captured by value. The buffer can outlive this storage, and the cache itself is process-global. - const String identity_key = makeObjectStorageIdentityCacheKey( - uri.endpoint, used_client->getRegion(), uri.bucket, object.remote_path); + const String identity_key = makeObjectStorageIdentityCacheKey(uri.endpoint, uri.bucket, object.remote_path); return std::make_unique( used_client, @@ -481,25 +480,11 @@ std::unique_ptr S3ObjectStorage::writeObject( /// NOLIN attributes, std::move(scheduler), disk_write_settings, - [identity_key](size_t size, std::optional etag) + [identity_key]() { - auto global_context = Context::getGlobalContextInstance(); - if (!global_context) - return; - auto cache = global_context->getObjectStorageIdentityCache(); - if (!cache) - return; - /// An empty ETag means the upload replaced the object but the counted size must not be cached. - if (!etag || etag->empty()) - { - cache->invalidate(identity_key); - return; - } - ObjectStorageIdentity identity; - identity.etag = std::move(*etag); - identity.size = size; - identity.is_size_known = true; - cache->replace(identity_key, std::move(identity)); + if (auto global_context = Context::getGlobalContextInstance()) + if (auto cache = global_context->getObjectStorageIdentityCache()) + cache->remove(identity_key); }); } @@ -1020,8 +1005,7 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool /// and the request is issued uncached. const bool identity_only = !with_tags; /// Bucket and path alone collide when two endpoints reuse a bucket name. - const String identity_key = makeObjectStorageIdentityCacheKey( - uri.endpoint, client.get()->getRegion(), uri.bucket, path); + const String identity_key = makeObjectStorageIdentityCacheKey(uri.endpoint, uri.bucket, path); ObjectStorageIdentityCachePtr identity_cache; if (identity_only) @@ -1047,11 +1031,6 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool } } - /// Captured after the miss and before the request. `setIfEpoch` refuses the result when a write - /// bumps the generation while this request is in flight, so the old size and ETag cannot replace - /// the identity published by that write. - const UInt64 identity_epoch = identity_cache ? identity_cache->epochOf(identity_key) : 0; - auto settings_ptr = s3_settings.get(); auto fetch = [&](const S3::Client & c) -> S3::ObjectInfo @@ -1119,10 +1098,9 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool } if (identity_cache) - identity_cache->setIfEpoch( + identity_cache->set( identity_key, - ObjectStorageIdentity{result.etag, result.size_bytes, result.is_size_known, result.part_offsets}, - identity_epoch); + ObjectStorageIdentity{result.etag, result.size_bytes, result.is_size_known, result.part_offsets}); return result; } @@ -1453,7 +1431,7 @@ void S3ObjectStorage::invalidateCachedIdentity(const std::string & path) const auto cache = global_context->getObjectStorageIdentityCache(); if (!cache) return; - cache->invalidate(makeObjectStorageIdentityCacheKey(uri.endpoint, client.get()->getRegion(), uri.bucket, path)); + cache->remove(makeObjectStorageIdentityCacheKey(uri.endpoint, uri.bucket, path)); } std::shared_ptr S3ObjectStorage::clientForRetryProfile(const ObjectStorageControlRequest & request) const diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h index e350bfa9527b7..2430e659798cc 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h @@ -249,8 +249,7 @@ class S3ObjectStorage : public IObjectStorage std::shared_ptr clientForRetryProfile(const ObjectStorageControlRequest & request) const; - /// Drop the identity-cache entry for `path`. The generation bump stops a HEAD that started - /// against the previous object from writing that identity back. + /// Drop the identity-cache entry for `path`. The next read of this key requests the metadata again. void invalidateCachedIdentity(const std::string & path) const; /// Runs `fn` and, if it failed because the vended credentials expired, refreshes this disk's diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 24f8e69154bd3..d6227c8c420cb 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -243,15 +243,14 @@ void WriteBufferFromS3::finalizeImpl() } catch (...) { - /// The upload already replaced the object. Tell the caller not to keep the previous identity, - /// and not to cache the size this buffer counted. + /// The upload already replaced the object. Drop any identity cached for the previous one. if (on_upload_complete) - on_upload_complete(0, std::nullopt); + on_upload_complete(); throw; } if (on_upload_complete) - on_upload_complete(total_size, object_etag); + on_upload_complete(); } void WriteBufferFromS3::cancelImpl() noexcept diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h index 63e5c37398ae0..ea1ebd11e5ed3 100644 --- a/src/IO/WriteBufferFromS3.h +++ b/src/IO/WriteBufferFromS3.h @@ -37,9 +37,8 @@ class TaskTracker; class WriteBufferFromS3 final : public WriteBufferFromFileBase { public: - /// Called after a successful upload, including when the post-upload size check fails. - /// `etag` is empty when the uploaded bytes must not be cached (the size check disagreed). - using OnUploadComplete = std::function etag)>; + /// Called after the upload has replaced the object, including when the post-upload size check fails. + using OnUploadComplete = std::function; WriteBufferFromS3( std::shared_ptr client_ptr_, diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 6845c38f1bfca..845a6d94a7a3a 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -4689,7 +4689,7 @@ void Context::clearMarkCache() const JemallocCacheArena::purge(); } -void Context::setObjectStorageIdentityCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio, UInt64 ttl_seconds) +void Context::setObjectStorageIdentityCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio) { std::lock_guard lock(shared->mutex); @@ -4701,7 +4701,7 @@ void Context::setObjectStorageIdentityCache(const String & cache_policy, size_t if (max_size_in_bytes == 0) return; - shared->object_storage_identity_cache = std::make_shared(cache_policy, max_size_in_bytes, size_ratio, ttl_seconds); + shared->object_storage_identity_cache = std::make_shared(cache_policy, max_size_in_bytes, size_ratio); } void Context::updateObjectStorageIdentityCacheConfiguration(const Poco::Util::AbstractConfiguration & config, size_t max_cache_size) @@ -4711,7 +4711,6 @@ void Context::updateObjectStorageIdentityCacheConfiguration(const Poco::Util::Ab size_t size = config.getUInt64("object_storage_identity_cache_size", 67108864); if (size > max_cache_size) size = max_cache_size; - const auto ttl_seconds = config.getUInt64("object_storage_identity_cache_ttl", 3600); /// Startup with size 0 leaves the pointer unset. A reload that raises the size has to create the /// cache here: returning would keep every read uncached until the next process start. @@ -4722,12 +4721,11 @@ void Context::updateObjectStorageIdentityCacheConfiguration(const Poco::Util::Ab const String policy = config.getString("object_storage_identity_cache_policy", "SLRU"); const double size_ratio = config.getDouble("object_storage_identity_cache_size_ratio", 0.5); - shared->object_storage_identity_cache = std::make_shared(policy, size, size_ratio, ttl_seconds); + shared->object_storage_identity_cache = std::make_shared(policy, size, size_ratio); return; } shared->object_storage_identity_cache->setMaxSizeInBytes(size); - shared->object_storage_identity_cache->setTTLSeconds(ttl_seconds); } std::shared_ptr Context::getObjectStorageIdentityCache() const diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 6bae698406dc1..94f971e71b975 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -1573,7 +1573,7 @@ class Context: public ContextData, public std::enable_shared_from_this /// Object-storage identity cache (per-object size/ETag/multipart-part-offsets) to avoid a HEAD /// on every object open. A max_size_in_bytes of 0 leaves the cache unset (disabled). - void setObjectStorageIdentityCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio, UInt64 ttl_seconds); + void setObjectStorageIdentityCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio); void updateObjectStorageIdentityCacheConfiguration(const Poco::Util::AbstractConfiguration & config, size_t max_cache_size); std::shared_ptr getObjectStorageIdentityCache() const; void clearObjectStorageIdentityCache() const; diff --git a/tests/queries/0_stateless/05486_object_storage_identity_cache_overwrite.sh b/tests/queries/0_stateless/05486_object_storage_identity_cache_overwrite.sh index 1ddc8b37ae55d..bc417617cccb2 100755 --- a/tests/queries/0_stateless/05486_object_storage_identity_cache_overwrite.sh +++ b/tests/queries/0_stateless/05486_object_storage_identity_cache_overwrite.sh @@ -4,10 +4,9 @@ # Tag no-parallel: the identity cache is process-global, and another test may clear it # Tag no-parallel-replicas: per-query S3 request counts change when a read is spread across replicas -# Overwriting an object must not leave the previous size and ETag in the identity cache. -# The upload stores the new size and ETag, so the following read is a hit and does not HEAD. -# The second insert is longer than the first. s3_validate_etag_on_read is off, so a stale -# cached size would be the read bound and the new rows would be truncated. +# Overwriting an object drops its identity-cache entry. The next read misses and HEADs, so it +# sees the new size. The second insert is longer than the first. s3_validate_etag_on_read is off, +# so a stale cached size would be the read bound and the new rows would be truncated. CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -90,11 +89,12 @@ insert_values() ${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear failed" insert_values "SELECT 1" -# The insert published the short object's identity, so this read does not HEAD. -read_and_check short $'1' eq0 ge1 eq0 +read_and_check short_cold $'1' ge1 eq0 ge1 +read_and_check short_warm $'1' eq0 ge1 eq0 -# No CLEAR between the inserts. The cache is holding the short object's size until the upload replaces it. +# No CLEAR. The upload drops the cached identity, so the next read misses and returns every new row. insert_values "SELECT 10 UNION ALL SELECT 20 UNION ALL SELECT 30" -read_and_check overwritten $'10\n20\n30' eq0 ge1 eq0 +read_and_check overwritten $'10\n20\n30' ge1 eq0 ge1 +read_and_check overwritten_warm $'10\n20\n30' eq0 ge1 eq0 echo OK From 25026d68b76b77635d3818b0500a51af45d813f4 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 17 Nov 2026 18:30:40 +0100 Subject: [PATCH 12/37] Object storage: keep a stale identity out of the cache after an overwrite A metadata read takes its HEAD before it stores the result. If this server overwrote, copied or deleted the object in that window, the read stored the size and ETag it fetched before the overwrite. The entry was dropped by the overwrite, so nothing removed the stale value again. Each identity-cache key now has a generation. An overwrite, copy or delete bumps it with `invalidate`, which also removes the entry. A metadata read captures the generation before its request and stores the result with `setIfEpoch`, which skips the store when the generation has changed. `clear` drops the generations together with the entries. Add the pause failpoint `object_storage_identity_cache_pause_before_store` between the fetch and the store, and test 05488 uses it to overwrite the object while a read is paused. The next read must miss and return the new rows. Co-authored-by: Cursor --- src/Common/FailPoint.cpp | 1 + .../ObjectStorageIdentityCache.cpp | 30 +++++- .../ObjectStorageIdentityCache.h | 26 ++++- .../ObjectStorages/S3/S3ObjectStorage.cpp | 21 +++- ...orage_identity_cache_stale_store.reference | 1 + ...ject_storage_identity_cache_stale_store.sh | 100 ++++++++++++++++++ 6 files changed, 171 insertions(+), 8 deletions(-) create mode 100644 tests/queries/0_stateless/05488_object_storage_identity_cache_stale_store.reference create mode 100755 tests/queries/0_stateless/05488_object_storage_identity_cache_stale_store.sh diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp index 4afe507ca35c2..c7c49cfe8c718 100644 --- a/src/Common/FailPoint.cpp +++ b/src/Common/FailPoint.cpp @@ -84,6 +84,7 @@ static struct InitFiu ONCE(s3_read_buffer_throw_expired_token) \ ONCE(s3_send_request_throw_expired_token) \ REGULAR(s3_read_inject_etag_mismatch) \ + PAUSEABLE_ONCE(object_storage_identity_cache_pause_before_store) \ REGULAR(azure_inject_forbidden_response) \ ONCE(azure_inject_forbidden_response_once) \ REGULAR(azure_inject_auth_failure_on_request) \ diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp index 7c6fc306ebaf7..d1645be5450ca 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp @@ -41,9 +41,37 @@ std::optional ObjectStorageIdentityCache::tryGet(const St return std::nullopt; } -void ObjectStorageIdentityCache::set(const String & key, ObjectStorageIdentity identity) +UInt64 ObjectStorageIdentityCache::epochOf(const String & key) const { + std::lock_guard lock(epoch_mutex); + const auto it = epoch_by_key.find(key); + if (it == epoch_by_key.end()) + return 0; + return it->second; +} + +void ObjectStorageIdentityCache::setIfEpoch(const String & key, ObjectStorageIdentity identity, UInt64 epoch) +{ + std::lock_guard lock(epoch_mutex); + const auto it = epoch_by_key.find(key); + const UInt64 current = it == epoch_by_key.end() ? 0 : it->second; + if (current != epoch) + return; Base::set(key, std::make_shared(std::move(identity))); } +void ObjectStorageIdentityCache::invalidate(const String & key) +{ + std::lock_guard lock(epoch_mutex); + ++epoch_by_key[key]; + Base::remove(key); +} + +void ObjectStorageIdentityCache::clear() +{ + std::lock_guard lock(epoch_mutex); + epoch_by_key.clear(); + Base::clear(); +} + } diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h index 079015a3d0ca8..3c47d808730d9 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h @@ -2,7 +2,9 @@ #include #include +#include +#include #include #include #include @@ -66,7 +68,9 @@ inline String makeObjectStorageIdentityCacheKey(std::string_view endpoint, std:: /// /// It is owned by the global Context (see Context::getObjectStorageIdentityCache), sized from server /// settings, and cleared by `SYSTEM DROP OBJECT STORAGE IDENTITY CACHE`. A write, copy, or delete by -/// this server removes the entry; the next read of that key requests the metadata again. Low-level +/// this server bumps that key's generation and removes the entry. A fetch stores its result only when +/// the generation is still the one it observed before the request, so a read that started earlier +/// cannot put the old identity back. The next read of that key requests the metadata again. Low-level /// object-storage code reaches the cache via Context::getGlobalContextInstance() and must tolerate a /// null pointer (no global context yet) by simply issuing the metadata request uncached. class ObjectStorageIdentityCache @@ -80,8 +84,24 @@ class ObjectStorageIdentityCache /// Look up an identity by `makeObjectStorageIdentityCacheKey`. Increments the hit/miss ProfileEvents. std::optional tryGet(const String & key); - /// Insert (or overwrite) the identity for a key. - void set(const String & key, ObjectStorageIdentity identity); + /// Generation of `key`. A fetch captures this before the request and passes it to `setIfEpoch`. + /// Zero when the key has never been invalidated. + UInt64 epochOf(const String & key) const; + + /// Store `identity` only when `epoch` is still the generation `epochOf` returned. + void setIfEpoch(const String & key, ObjectStorageIdentity identity, UInt64 epoch); + + /// Bump the generation and drop the entry. A fetch already in flight will not put the old identity back. + void invalidate(const String & key); + + /// Drop every entry and generation. Hides `CacheBase::clear` so a generation cannot outlive its entry. + void clear(); + +private: + mutable std::mutex epoch_mutex; + /// Generations of keys this server has replaced or deleted. Absent means generation 0. + /// Guarded by `epoch_mutex`. An invalidated key stays here so a late fetch can see the bump. + UnorderedMapWithMemoryTracking epoch_by_key; }; using ObjectStorageIdentityCachePtr = std::shared_ptr; diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp index 21bff1cdcb61a..61c6d9fbd6447 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp @@ -62,6 +62,7 @@ namespace CurrentMetrics namespace DB::FailPoints { extern const char object_storage_force_refresh_callback_success[]; + extern const char object_storage_identity_cache_pause_before_store[]; } @@ -484,7 +485,7 @@ std::unique_ptr S3ObjectStorage::writeObject( /// NOLIN { if (auto global_context = Context::getGlobalContextInstance()) if (auto cache = global_context->getObjectStorageIdentityCache()) - cache->remove(identity_key); + cache->invalidate(identity_key); }); } @@ -1031,6 +1032,12 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool } } + /// Captured before the request. `setIfEpoch` stores the result only while this generation still + /// matches, so an overwrite that lands during the request cannot be undone by this fetch. + UInt64 identity_epoch = 0; + if (identity_cache) + identity_epoch = identity_cache->epochOf(identity_key); + auto settings_ptr = s3_settings.get(); auto fetch = [&](const S3::Client & c) -> S3::ObjectInfo @@ -1098,9 +1105,15 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool } if (identity_cache) - identity_cache->set( + { + /// The read that reached this point already holds the identity from before any overwrite that + /// happened while it was paused. Resuming must not store that identity. + FailPointInjection::pauseFailPoint(FailPoints::object_storage_identity_cache_pause_before_store); + identity_cache->setIfEpoch( identity_key, - ObjectStorageIdentity{result.etag, result.size_bytes, result.is_size_known, result.part_offsets}); + ObjectStorageIdentity{result.etag, result.size_bytes, result.is_size_known, result.part_offsets}, + identity_epoch); + } return result; } @@ -1431,7 +1444,7 @@ void S3ObjectStorage::invalidateCachedIdentity(const std::string & path) const auto cache = global_context->getObjectStorageIdentityCache(); if (!cache) return; - cache->remove(makeObjectStorageIdentityCacheKey(uri.endpoint, uri.bucket, path)); + cache->invalidate(makeObjectStorageIdentityCacheKey(uri.endpoint, uri.bucket, path)); } std::shared_ptr S3ObjectStorage::clientForRetryProfile(const ObjectStorageControlRequest & request) const diff --git a/tests/queries/0_stateless/05488_object_storage_identity_cache_stale_store.reference b/tests/queries/0_stateless/05488_object_storage_identity_cache_stale_store.reference new file mode 100644 index 0000000000000..d86bac9de59ab --- /dev/null +++ b/tests/queries/0_stateless/05488_object_storage_identity_cache_stale_store.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/05488_object_storage_identity_cache_stale_store.sh b/tests/queries/0_stateless/05488_object_storage_identity_cache_stale_store.sh new file mode 100755 index 0000000000000..d2a04cee8d359 --- /dev/null +++ b/tests/queries/0_stateless/05488_object_storage_identity_cache_stale_store.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-parallel-replicas +# Tag no-fasttest: reads a file through the mock S3 server at localhost:11111 +# Tag no-parallel: the identity cache and the pause failpoint are process-global +# Tag no-parallel-replicas: per-query S3 request counts change when a read is spread across replicas + +# A metadata read is paused after its HEAD and before it stores the identity. An overwrite in that +# window bumps the key's generation. When the read resumes it must not store the identity it fetched, +# so the next read misses and returns the new rows. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +URL="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" +READ_SETTINGS="max_threads = 1, use_hive_partitioning = 0, parallel_replicas_for_cluster_engines = 0, enable_filesystem_cache = 0, use_page_cache_for_object_storage = 0, s3_validate_etag_on_read = 0, use_object_storage_list_objects_cache = 0" +FAILPOINT="object_storage_identity_cache_pause_before_store" + +fail() +{ + echo "FAIL: $*" >&2 + exit 1 +} + +cleanup() +{ + ${CLICKHOUSE_CLIENT} -q "SYSTEM DISABLE FAILPOINT ${FAILPOINT}" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +check_count() +{ + local label=$1 + local name=$2 + local got=$3 + local mode=$4 + case "$mode" in + eq0) [[ $got == "0" ]] || fail "${label}: ${name}=${got}, expected 0" ;; + ge1) [[ $got -ge 1 ]] || fail "${label}: ${name}=${got}, expected >= 1" ;; + *) fail "${label}: bad mode ${mode}" ;; + esac +} + +insert_values() +{ + local values=$1 + ${CLICKHOUSE_CLIENT} -q " + INSERT INTO FUNCTION s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') + ${values} + SETTINGS s3_truncate_on_insert = 1" || fail "insert failed: ${values}" +} + +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear failed" + +insert_values "SELECT 1" + +${CLICKHOUSE_CLIENT} -q "SYSTEM ENABLE FAILPOINT ${FAILPOINT}" || fail "enable failpoint failed" + +# The read misses, HEADs the current object, and pauses before storing that identity. +${CLICKHOUSE_CLIENT} -q \ + "SELECT n FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') ORDER BY n SETTINGS ${READ_SETTINGS} FORMAT TabSeparated" \ + >/dev/null & +reader_pid=$! + +${CLICKHOUSE_CLIENT} -q "SYSTEM WAIT FAILPOINT ${FAILPOINT} PAUSE" || fail "wait for pause failed" + +insert_values "SELECT 10 UNION ALL SELECT 20 UNION ALL SELECT 30" + +${CLICKHOUSE_CLIENT} -q "SYSTEM DISABLE FAILPOINT ${FAILPOINT}" || fail "disable failpoint failed" +wait "$reader_pid" || fail "in-flight read failed" + +qid="idcache_stale_${CLICKHOUSE_DATABASE}" +data=$(${CLICKHOUSE_CLIENT} --query_id "$qid" -q \ + "SELECT n FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') ORDER BY n SETTINGS ${READ_SETTINGS} FORMAT TabSeparated") \ + || fail "check select failed" +[[ $data == $'10\n20\n30' ]] || fail "data '${data}', expected the three new rows" + +${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "flush logs failed" +row=$(${CLICKHOUSE_CLIENT} -q " + SELECT + ProfileEvents['S3HeadObject'], + ProfileEvents['ObjectStorageIdentityCacheHits'], + ProfileEvents['ObjectStorageIdentityCacheMisses'] + FROM system.query_log + WHERE event_date >= yesterday() + AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND query_id = '${qid}' + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC + LIMIT 1 + FORMAT TabSeparated") || fail "query_log failed" +[[ -n $row ]] || fail "no query_log row" + +IFS=$'\t' read -r head hits misses <<< "$row" +check_count check S3HeadObject "$head" ge1 +check_count check Hits "$hits" eq0 +check_count check Misses "$misses" ge1 + +echo OK From 5c9ae9901d2321842b8c7b99fe36d4b7e3f9568e Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 17 Nov 2026 18:36:24 +0100 Subject: [PATCH 13/37] Object storage: keep a fetch from storing after the identity cache is cleared `SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE` removed the per-key generations along with the entries. A key that had been overwritten therefore looked untouched again, and a metadata read that started before the clear and resumed after it could store the size and ETag it had fetched before the overwrite. `clear` now bumps a cache-wide generation and does not reset it. The fetch captures it together with the per-key generation as the new `ObjectStorageIdentityEpoch`, and `setIfEpoch` stores only when both still match. Test 05489 pauses a read after its HEAD, overwrites the object, clears the cache, and resumes the read. The next read must miss and return the new rows. Co-authored-by: Cursor --- .../ObjectStorageIdentityCache.cpp | 11 +- .../ObjectStorageIdentityCache.h | 29 +++-- .../ObjectStorages/S3/S3ObjectStorage.cpp | 2 +- ...dentity_cache_clear_during_fetch.reference | 1 + ...orage_identity_cache_clear_during_fetch.sh | 104 ++++++++++++++++++ 5 files changed, 133 insertions(+), 14 deletions(-) create mode 100644 tests/queries/0_stateless/05489_object_storage_identity_cache_clear_during_fetch.reference create mode 100755 tests/queries/0_stateless/05489_object_storage_identity_cache_clear_during_fetch.sh diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp index d1645be5450ca..8b49fa18b5bb0 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp @@ -41,20 +41,18 @@ std::optional ObjectStorageIdentityCache::tryGet(const St return std::nullopt; } -UInt64 ObjectStorageIdentityCache::epochOf(const String & key) const +ObjectStorageIdentityEpoch ObjectStorageIdentityCache::epochOf(const String & key) const { std::lock_guard lock(epoch_mutex); const auto it = epoch_by_key.find(key); - if (it == epoch_by_key.end()) - return 0; - return it->second; + return {clear_epoch, it == epoch_by_key.end() ? 0 : it->second}; } -void ObjectStorageIdentityCache::setIfEpoch(const String & key, ObjectStorageIdentity identity, UInt64 epoch) +void ObjectStorageIdentityCache::setIfEpoch(const String & key, ObjectStorageIdentity identity, ObjectStorageIdentityEpoch epoch) { std::lock_guard lock(epoch_mutex); const auto it = epoch_by_key.find(key); - const UInt64 current = it == epoch_by_key.end() ? 0 : it->second; + const ObjectStorageIdentityEpoch current{clear_epoch, it == epoch_by_key.end() ? 0 : it->second}; if (current != epoch) return; Base::set(key, std::make_shared(std::move(identity))); @@ -70,6 +68,7 @@ void ObjectStorageIdentityCache::invalidate(const String & key) void ObjectStorageIdentityCache::clear() { std::lock_guard lock(epoch_mutex); + ++clear_epoch; epoch_by_key.clear(); Base::clear(); } diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h index 3c47d808730d9..f283e57e69cee 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h @@ -61,6 +61,18 @@ inline String makeObjectStorageIdentityCacheKey(std::string_view endpoint, std:: return key; } +/// What a metadata fetch observed before its request. `key` counts the writes, copies, and deletes of +/// one object. `clear` counts the times the whole cache was dropped: `clear` also forgets every `key` +/// counter, so without it a key would look untouched again and a fetch in flight across the drop +/// would store the identity it fetched before the overwrite. +struct ObjectStorageIdentityEpoch +{ + UInt64 clear = 0; + UInt64 key = 0; + + bool operator==(const ObjectStorageIdentityEpoch &) const = default; +}; + /// Cache of object-storage identity, keyed by endpoint, bucket and object path, so opening an object /// does not require a fresh HEAD / GetObjectAttributes on every read (a lake scan otherwise re-HEADs /// every candidate file on every query). Backed by CacheBase (LRU/SLRU) for size-bounded eviction @@ -68,9 +80,10 @@ inline String makeObjectStorageIdentityCacheKey(std::string_view endpoint, std:: /// /// It is owned by the global Context (see Context::getObjectStorageIdentityCache), sized from server /// settings, and cleared by `SYSTEM DROP OBJECT STORAGE IDENTITY CACHE`. A write, copy, or delete by -/// this server bumps that key's generation and removes the entry. A fetch stores its result only when -/// the generation is still the one it observed before the request, so a read that started earlier -/// cannot put the old identity back. The next read of that key requests the metadata again. Low-level +/// this server bumps that key's generation and removes the entry, and `SYSTEM DROP` bumps a cache-wide +/// generation. A fetch stores its result only when both are still the ones it observed before the +/// request, so a read that started earlier cannot put the old identity back. The next read of that key +/// requests the metadata again. Low-level /// object-storage code reaches the cache via Context::getGlobalContextInstance() and must tolerate a /// null pointer (no global context yet) by simply issuing the metadata request uncached. class ObjectStorageIdentityCache @@ -85,20 +98,22 @@ class ObjectStorageIdentityCache std::optional tryGet(const String & key); /// Generation of `key`. A fetch captures this before the request and passes it to `setIfEpoch`. - /// Zero when the key has never been invalidated. - UInt64 epochOf(const String & key) const; + ObjectStorageIdentityEpoch epochOf(const String & key) const; /// Store `identity` only when `epoch` is still the generation `epochOf` returned. - void setIfEpoch(const String & key, ObjectStorageIdentity identity, UInt64 epoch); + void setIfEpoch(const String & key, ObjectStorageIdentity identity, ObjectStorageIdentityEpoch epoch); /// Bump the generation and drop the entry. A fetch already in flight will not put the old identity back. void invalidate(const String & key); - /// Drop every entry and generation. Hides `CacheBase::clear` so a generation cannot outlive its entry. + /// Drop every entry and per-key generation, and bump the cache-wide generation. Hides + /// `CacheBase::clear` so a fetch in flight across the drop cannot store afterwards. void clear(); private: mutable std::mutex epoch_mutex; + /// Times `clear` ran. Guarded by `epoch_mutex`. + UInt64 clear_epoch = 0; /// Generations of keys this server has replaced or deleted. Absent means generation 0. /// Guarded by `epoch_mutex`. An invalidated key stays here so a late fetch can see the bump. UnorderedMapWithMemoryTracking epoch_by_key; diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp index 61c6d9fbd6447..d163fa82bcd6b 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp @@ -1034,7 +1034,7 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool /// Captured before the request. `setIfEpoch` stores the result only while this generation still /// matches, so an overwrite that lands during the request cannot be undone by this fetch. - UInt64 identity_epoch = 0; + ObjectStorageIdentityEpoch identity_epoch; if (identity_cache) identity_epoch = identity_cache->epochOf(identity_key); diff --git a/tests/queries/0_stateless/05489_object_storage_identity_cache_clear_during_fetch.reference b/tests/queries/0_stateless/05489_object_storage_identity_cache_clear_during_fetch.reference new file mode 100644 index 0000000000000..d86bac9de59ab --- /dev/null +++ b/tests/queries/0_stateless/05489_object_storage_identity_cache_clear_during_fetch.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/05489_object_storage_identity_cache_clear_during_fetch.sh b/tests/queries/0_stateless/05489_object_storage_identity_cache_clear_during_fetch.sh new file mode 100755 index 0000000000000..c1d183c0dc4b9 --- /dev/null +++ b/tests/queries/0_stateless/05489_object_storage_identity_cache_clear_during_fetch.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-parallel-replicas +# Tag no-fasttest: reads a file through the mock S3 server at localhost:11111 +# Tag no-parallel: the identity cache and the pause failpoint are process-global +# Tag no-parallel-replicas: per-query S3 request counts change when a read is spread across replicas + +# A metadata read is paused after its HEAD and before it stores the identity. The object is overwritten +# and then the whole cache is dropped. Dropping the cache forgets the per-key generations, so the key +# looks untouched again. When the read resumes it must still not store the identity it fetched before +# the overwrite: the next read misses and returns the new rows. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +URL="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" +READ_SETTINGS="max_threads = 1, use_hive_partitioning = 0, parallel_replicas_for_cluster_engines = 0, enable_filesystem_cache = 0, use_page_cache_for_object_storage = 0, s3_validate_etag_on_read = 0, use_object_storage_list_objects_cache = 0" +FAILPOINT="object_storage_identity_cache_pause_before_store" + +fail() +{ + echo "FAIL: $*" >&2 + exit 1 +} + +cleanup() +{ + ${CLICKHOUSE_CLIENT} -q "SYSTEM DISABLE FAILPOINT ${FAILPOINT}" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +check_count() +{ + local label=$1 + local name=$2 + local got=$3 + local mode=$4 + case "$mode" in + eq0) [[ $got == "0" ]] || fail "${label}: ${name}=${got}, expected 0" ;; + ge1) [[ $got -ge 1 ]] || fail "${label}: ${name}=${got}, expected >= 1" ;; + *) fail "${label}: bad mode ${mode}" ;; + esac +} + +insert_values() +{ + local values=$1 + ${CLICKHOUSE_CLIENT} -q " + INSERT INTO FUNCTION s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') + ${values} + SETTINGS s3_truncate_on_insert = 1" || fail "insert failed: ${values}" +} + +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear failed" + +insert_values "SELECT 1" + +${CLICKHOUSE_CLIENT} -q "SYSTEM ENABLE FAILPOINT ${FAILPOINT}" || fail "enable failpoint failed" + +# The read misses, HEADs the current object, and pauses before storing that identity. +${CLICKHOUSE_CLIENT} -q \ + "SELECT n FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') ORDER BY n SETTINGS ${READ_SETTINGS} FORMAT TabSeparated" \ + >/dev/null & +reader_pid=$! + +${CLICKHOUSE_CLIENT} -q "SYSTEM WAIT FAILPOINT ${FAILPOINT} PAUSE" || fail "wait for pause failed" + +insert_values "SELECT 10 UNION ALL SELECT 20 UNION ALL SELECT 30" + +# Drops every entry and every per-key generation while the read is still paused. +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear during fetch failed" + +${CLICKHOUSE_CLIENT} -q "SYSTEM DISABLE FAILPOINT ${FAILPOINT}" || fail "disable failpoint failed" +wait "$reader_pid" || fail "in-flight read failed" + +qid="idcache_clear_${CLICKHOUSE_DATABASE}" +data=$(${CLICKHOUSE_CLIENT} --query_id "$qid" -q \ + "SELECT n FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') ORDER BY n SETTINGS ${READ_SETTINGS} FORMAT TabSeparated") \ + || fail "check select failed" +[[ $data == $'10\n20\n30' ]] || fail "data '${data}', expected the three new rows" + +${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "flush logs failed" +row=$(${CLICKHOUSE_CLIENT} -q " + SELECT + ProfileEvents['S3HeadObject'], + ProfileEvents['ObjectStorageIdentityCacheHits'], + ProfileEvents['ObjectStorageIdentityCacheMisses'] + FROM system.query_log + WHERE event_date >= yesterday() + AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND query_id = '${qid}' + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC + LIMIT 1 + FORMAT TabSeparated") || fail "query_log failed" +[[ -n $row ]] || fail "no query_log row" + +IFS=$'\t' read -r head hits misses <<< "$row" +check_count check S3HeadObject "$head" ge1 +check_count check Hits "$hits" eq0 +check_count check Misses "$misses" ge1 + +echo OK From 7099ed79540923c457a73091d49133606b384509 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 17 Nov 2026 18:41:01 +0100 Subject: [PATCH 14/37] Object storage: cap the identity cache generation map Each `invalidate` of a key left a per-key generation record in `epoch_by_key`, and nothing removed it except `clear`. The map grew by one entry for every overwritten, copied or deleted object for the life of the process, outside the cache's size limit. The map is now accounted in bytes and limited to the cache's configured size. When a new record would exceed that limit, the map is dropped and the cache-wide generation is bumped, so every fetch already in flight fails its `setIfEpoch` check and does not store. One record is always kept, so the map never refuses a key even when that single record is larger than the limit. Co-authored-by: Cursor --- .../ObjectStorageIdentityCache.cpp | 36 +++++++++++++++++-- .../ObjectStorageIdentityCache.h | 27 +++++++++++--- 2 files changed, 55 insertions(+), 8 deletions(-) diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp index 8b49fa18b5bb0..d87dd3ef4081e 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp @@ -58,18 +58,48 @@ void ObjectStorageIdentityCache::setIfEpoch(const String & key, ObjectStorageIde Base::set(key, std::make_shared(std::move(identity))); } +size_t ObjectStorageIdentityCache::generationRecordBytes(const String & key) +{ + return sizeof(String) + key.size() + sizeof(UInt64) + 64; +} + +void ObjectStorageIdentityCache::forgetPerKeyGenerations() +{ + ++clear_epoch; + epoch_by_key.clear(); + generation_bytes = 0; +} + +void ObjectStorageIdentityCache::reserveGenerationBytes(size_t extra) +{ + const size_t budget = Base::maxSizeInBytes(); + if (generation_bytes <= budget && extra <= budget - generation_bytes) + return; + forgetPerKeyGenerations(); +} + void ObjectStorageIdentityCache::invalidate(const String & key) { std::lock_guard lock(epoch_mutex); - ++epoch_by_key[key]; + const auto it = epoch_by_key.find(key); + if (it == epoch_by_key.end()) + { + const size_t extra = generationRecordBytes(key); + reserveGenerationBytes(extra); + epoch_by_key.emplace(key, 1); + generation_bytes += extra; + } + else + { + ++it->second; + } Base::remove(key); } void ObjectStorageIdentityCache::clear() { std::lock_guard lock(epoch_mutex); - ++clear_epoch; - epoch_by_key.clear(); + forgetPerKeyGenerations(); Base::clear(); } diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h index f283e57e69cee..de312b53bb553 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h @@ -81,9 +81,11 @@ struct ObjectStorageIdentityEpoch /// It is owned by the global Context (see Context::getObjectStorageIdentityCache), sized from server /// settings, and cleared by `SYSTEM DROP OBJECT STORAGE IDENTITY CACHE`. A write, copy, or delete by /// this server bumps that key's generation and removes the entry, and `SYSTEM DROP` bumps a cache-wide -/// generation. A fetch stores its result only when both are still the ones it observed before the -/// request, so a read that started earlier cannot put the old identity back. The next read of that key -/// requests the metadata again. Low-level +/// generation. The per-key generations are kept only up to the cache's configured size. Past that, the +/// map is dropped and the cache-wide generation is bumped, which rejects every fetch already in flight. +/// A fetch stores its result only when both generations are still the ones it +/// observed before the request, so a read that started earlier cannot put the old identity back. The +/// next read of that key requests the metadata again. Low-level /// object-storage code reaches the cache via Context::getGlobalContextInstance() and must tolerate a /// null pointer (no global context yet) by simply issuing the metadata request uncached. class ObjectStorageIdentityCache @@ -111,12 +113,27 @@ class ObjectStorageIdentityCache void clear(); private: + /// Bytes of one per-key generation record: the key, its counter, and a hashmap node. + static size_t generationRecordBytes(const String & key); + + /// `epoch_mutex` must be held. Forget every per-key generation and bump the cache-wide generation. + void forgetPerKeyGenerations(); + + /// `epoch_mutex` must be held. When `extra` bytes would take the generation map past the cache's + /// configured size, forget the per-key generations first. One new record is then kept even if that + /// record alone is larger than the configured size. + void reserveGenerationBytes(size_t extra); + mutable std::mutex epoch_mutex; - /// Times `clear` ran. Guarded by `epoch_mutex`. + /// Times the per-key generations were forgotten, by `clear` or because the map reached its size. + /// Guarded by `epoch_mutex`. UInt64 clear_epoch = 0; /// Generations of keys this server has replaced or deleted. Absent means generation 0. - /// Guarded by `epoch_mutex`. An invalidated key stays here so a late fetch can see the bump. + /// Guarded by `epoch_mutex`. A key stays here so a fetch that started earlier can see the bump, + /// until the map reaches the cache's configured size. UnorderedMapWithMemoryTracking epoch_by_key; + /// `epoch_by_key` accounted with `generationRecordBytes`. Guarded by `epoch_mutex`. + size_t generation_bytes = 0; }; using ObjectStorageIdentityCachePtr = std::shared_ptr; From a1580caae094bfda4e6adec7c42d17693a0f4834 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 17 Nov 2026 18:49:51 +0100 Subject: [PATCH 15/37] Object storage: simplify the identity cache generation to one epoch The per-key generation map added to keep stale identities out of the identity cache was large and needed its own byte accounting and cap. A single cache-wide epoch covers the same cases with much less code. `invalidate` (overwrite, copy or delete) and `clear` advance the epoch and `invalidate` also removes the entry. A metadata read captures the epoch before its request and stores the result with `setIfEpoch`, which skips the store when the epoch has changed. The epoch is never reset, so a fetch in flight cannot match a later epoch by accident. The trade-off is that any write, copy or delete of any object makes every metadata fetch in flight skip its store. The fetch still returns its result to the caller, and the next read of the key fetches again. Tests 05488 and 05489 keep their scenarios. Only their comments change. Co-authored-by: Cursor --- .../ObjectStorageIdentityCache.cpp | 46 ++------------ .../ObjectStorageIdentityCache.h | 63 +++++-------------- .../ObjectStorages/S3/S3ObjectStorage.cpp | 8 +-- ...ject_storage_identity_cache_stale_store.sh | 2 +- ...orage_identity_cache_clear_during_fetch.sh | 4 +- 5 files changed, 28 insertions(+), 95 deletions(-) diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp index d87dd3ef4081e..30bedf3855cd4 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp @@ -41,65 +41,31 @@ std::optional ObjectStorageIdentityCache::tryGet(const St return std::nullopt; } -ObjectStorageIdentityEpoch ObjectStorageIdentityCache::epochOf(const String & key) const +UInt64 ObjectStorageIdentityCache::currentEpoch() const { std::lock_guard lock(epoch_mutex); - const auto it = epoch_by_key.find(key); - return {clear_epoch, it == epoch_by_key.end() ? 0 : it->second}; + return epoch; } -void ObjectStorageIdentityCache::setIfEpoch(const String & key, ObjectStorageIdentity identity, ObjectStorageIdentityEpoch epoch) +void ObjectStorageIdentityCache::setIfEpoch(const String & key, ObjectStorageIdentity identity, UInt64 expected_epoch) { std::lock_guard lock(epoch_mutex); - const auto it = epoch_by_key.find(key); - const ObjectStorageIdentityEpoch current{clear_epoch, it == epoch_by_key.end() ? 0 : it->second}; - if (current != epoch) + if (epoch != expected_epoch) return; Base::set(key, std::make_shared(std::move(identity))); } -size_t ObjectStorageIdentityCache::generationRecordBytes(const String & key) -{ - return sizeof(String) + key.size() + sizeof(UInt64) + 64; -} - -void ObjectStorageIdentityCache::forgetPerKeyGenerations() -{ - ++clear_epoch; - epoch_by_key.clear(); - generation_bytes = 0; -} - -void ObjectStorageIdentityCache::reserveGenerationBytes(size_t extra) -{ - const size_t budget = Base::maxSizeInBytes(); - if (generation_bytes <= budget && extra <= budget - generation_bytes) - return; - forgetPerKeyGenerations(); -} - void ObjectStorageIdentityCache::invalidate(const String & key) { std::lock_guard lock(epoch_mutex); - const auto it = epoch_by_key.find(key); - if (it == epoch_by_key.end()) - { - const size_t extra = generationRecordBytes(key); - reserveGenerationBytes(extra); - epoch_by_key.emplace(key, 1); - generation_bytes += extra; - } - else - { - ++it->second; - } + ++epoch; Base::remove(key); } void ObjectStorageIdentityCache::clear() { std::lock_guard lock(epoch_mutex); - forgetPerKeyGenerations(); + ++epoch; Base::clear(); } diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h index de312b53bb553..65fb0abc104e1 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h @@ -2,7 +2,6 @@ #include #include -#include #include #include @@ -61,18 +60,6 @@ inline String makeObjectStorageIdentityCacheKey(std::string_view endpoint, std:: return key; } -/// What a metadata fetch observed before its request. `key` counts the writes, copies, and deletes of -/// one object. `clear` counts the times the whole cache was dropped: `clear` also forgets every `key` -/// counter, so without it a key would look untouched again and a fetch in flight across the drop -/// would store the identity it fetched before the overwrite. -struct ObjectStorageIdentityEpoch -{ - UInt64 clear = 0; - UInt64 key = 0; - - bool operator==(const ObjectStorageIdentityEpoch &) const = default; -}; - /// Cache of object-storage identity, keyed by endpoint, bucket and object path, so opening an object /// does not require a fresh HEAD / GetObjectAttributes on every read (a lake scan otherwise re-HEADs /// every candidate file on every query). Backed by CacheBase (LRU/SLRU) for size-bounded eviction @@ -80,14 +67,11 @@ struct ObjectStorageIdentityEpoch /// /// It is owned by the global Context (see Context::getObjectStorageIdentityCache), sized from server /// settings, and cleared by `SYSTEM DROP OBJECT STORAGE IDENTITY CACHE`. A write, copy, or delete by -/// this server bumps that key's generation and removes the entry, and `SYSTEM DROP` bumps a cache-wide -/// generation. The per-key generations are kept only up to the cache's configured size. Past that, the -/// map is dropped and the cache-wide generation is bumped, which rejects every fetch already in flight. -/// A fetch stores its result only when both generations are still the ones it -/// observed before the request, so a read that started earlier cannot put the old identity back. The -/// next read of that key requests the metadata again. Low-level -/// object-storage code reaches the cache via Context::getGlobalContextInstance() and must tolerate a -/// null pointer (no global context yet) by simply issuing the metadata request uncached. +/// this server removes the entry and advances the epoch. A metadata fetch reads the epoch before its +/// request and stores its result only if the epoch is unchanged, so a fetch that started before a +/// replacement cannot put the old identity back. The next read of that key requests the metadata again. +/// Low-level object-storage code reaches the cache via Context::getGlobalContextInstance() and must +/// tolerate a null pointer (no global context yet) by simply issuing the metadata request uncached. class ObjectStorageIdentityCache : public CacheBase, ObjectStorageIdentityWeight> { @@ -99,41 +83,24 @@ class ObjectStorageIdentityCache /// Look up an identity by `makeObjectStorageIdentityCacheKey`. Increments the hit/miss ProfileEvents. std::optional tryGet(const String & key); - /// Generation of `key`. A fetch captures this before the request and passes it to `setIfEpoch`. - ObjectStorageIdentityEpoch epochOf(const String & key) const; + /// The current epoch. A fetch captures this before the request and passes it to `setIfEpoch`. + UInt64 currentEpoch() const; - /// Store `identity` only when `epoch` is still the generation `epochOf` returned. - void setIfEpoch(const String & key, ObjectStorageIdentity identity, ObjectStorageIdentityEpoch epoch); + /// Store `identity` only when `epoch` is still the epoch `currentEpoch` returned. + void setIfEpoch(const String & key, ObjectStorageIdentity identity, UInt64 epoch); - /// Bump the generation and drop the entry. A fetch already in flight will not put the old identity back. + /// Advance the epoch and drop the entry. A fetch already in flight will not put the old identity back. void invalidate(const String & key); - /// Drop every entry and per-key generation, and bump the cache-wide generation. Hides - /// `CacheBase::clear` so a fetch in flight across the drop cannot store afterwards. + /// Drop every entry and advance the epoch. Hides `CacheBase::clear` so a fetch in flight across the + /// drop cannot store afterwards. void clear(); private: - /// Bytes of one per-key generation record: the key, its counter, and a hashmap node. - static size_t generationRecordBytes(const String & key); - - /// `epoch_mutex` must be held. Forget every per-key generation and bump the cache-wide generation. - void forgetPerKeyGenerations(); - - /// `epoch_mutex` must be held. When `extra` bytes would take the generation map past the cache's - /// configured size, forget the per-key generations first. One new record is then kept even if that - /// record alone is larger than the configured size. - void reserveGenerationBytes(size_t extra); - mutable std::mutex epoch_mutex; - /// Times the per-key generations were forgotten, by `clear` or because the map reached its size. - /// Guarded by `epoch_mutex`. - UInt64 clear_epoch = 0; - /// Generations of keys this server has replaced or deleted. Absent means generation 0. - /// Guarded by `epoch_mutex`. A key stays here so a fetch that started earlier can see the bump, - /// until the map reaches the cache's configured size. - UnorderedMapWithMemoryTracking epoch_by_key; - /// `epoch_by_key` accounted with `generationRecordBytes`. Guarded by `epoch_mutex`. - size_t generation_bytes = 0; + /// Advanced by every `invalidate` and `clear`. Guarded by `epoch_mutex`. It is never reset, so a + /// fetch in flight can never match a later epoch by accident. + UInt64 epoch = 0; }; using ObjectStorageIdentityCachePtr = std::shared_ptr; diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp index d163fa82bcd6b..3065ac805178b 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp @@ -1032,11 +1032,11 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool } } - /// Captured before the request. `setIfEpoch` stores the result only while this generation still - /// matches, so an overwrite that lands during the request cannot be undone by this fetch. - ObjectStorageIdentityEpoch identity_epoch; + /// Captured before the request. `setIfEpoch` stores the result only while the epoch is unchanged, + /// so a write, copy, delete, or drop that lands during the request cannot be undone by this fetch. + UInt64 identity_epoch = 0; if (identity_cache) - identity_epoch = identity_cache->epochOf(identity_key); + identity_epoch = identity_cache->currentEpoch(); auto settings_ptr = s3_settings.get(); diff --git a/tests/queries/0_stateless/05488_object_storage_identity_cache_stale_store.sh b/tests/queries/0_stateless/05488_object_storage_identity_cache_stale_store.sh index d2a04cee8d359..99b5c6c11a035 100755 --- a/tests/queries/0_stateless/05488_object_storage_identity_cache_stale_store.sh +++ b/tests/queries/0_stateless/05488_object_storage_identity_cache_stale_store.sh @@ -5,7 +5,7 @@ # Tag no-parallel-replicas: per-query S3 request counts change when a read is spread across replicas # A metadata read is paused after its HEAD and before it stores the identity. An overwrite in that -# window bumps the key's generation. When the read resumes it must not store the identity it fetched, +# window advances the cache epoch. When the read resumes it must not store the identity it fetched, # so the next read misses and returns the new rows. CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) diff --git a/tests/queries/0_stateless/05489_object_storage_identity_cache_clear_during_fetch.sh b/tests/queries/0_stateless/05489_object_storage_identity_cache_clear_during_fetch.sh index c1d183c0dc4b9..0cc7405e05bbe 100755 --- a/tests/queries/0_stateless/05489_object_storage_identity_cache_clear_during_fetch.sh +++ b/tests/queries/0_stateless/05489_object_storage_identity_cache_clear_during_fetch.sh @@ -5,7 +5,7 @@ # Tag no-parallel-replicas: per-query S3 request counts change when a read is spread across replicas # A metadata read is paused after its HEAD and before it stores the identity. The object is overwritten -# and then the whole cache is dropped. Dropping the cache forgets the per-key generations, so the key +# and then the whole cache is dropped. The drop advances the cache epoch, so the read must not store # looks untouched again. When the read resumes it must still not store the identity it fetched before # the overwrite: the next read misses and returns the new rows. @@ -67,7 +67,7 @@ ${CLICKHOUSE_CLIENT} -q "SYSTEM WAIT FAILPOINT ${FAILPOINT} PAUSE" || fail "wait insert_values "SELECT 10 UNION ALL SELECT 20 UNION ALL SELECT 30" -# Drops every entry and every per-key generation while the read is still paused. +# Drops every entry and advances the epoch while the read is still paused. ${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear during fetch failed" ${CLICKHOUSE_CLIENT} -q "SYSTEM DISABLE FAILPOINT ${FAILPOINT}" || fail "disable failpoint failed" From 46e8d1c629007eef75766501ede2c014de8cf6da Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 17 Nov 2026 19:13:01 +0100 Subject: [PATCH 16/37] Object storage: drop a replaced object's identity from the identity cache A read that found the object replaced kept the cached size and ETag. The next read was pinned to that ETag and failed with `S3_OBJECT_CHANGED_DURING_READ` again, and every retry hit the same entry. Nothing in the read path removed it, because a replace made outside this server does not pass through `S3ObjectStorage::writeObject`. `ReadBufferFromS3` now calls a callback when a GET fails `If-Match` or, for an unversioned read, returns a not-found error. `S3ObjectStorage::readObject` passes a callback that invalidates the key, which also advances the cache epoch so a fetch already in flight cannot store the old identity back. `ObjectStorageIdentityCache` also orders stores per key. `beginFetch` hands out a sequence number, and `setIfEpoch` skips a store when a later fetch of the same key has already stored. An older observation therefore cannot overwrite a newer one. Test 05490 replaces the object with a plain PUT, expects the read to fail with `S3_OBJECT_CHANGED_DURING_READ`, and checks that the retry misses, sends a HEAD, and returns the new rows. Co-authored-by: Cursor --- .../ObjectStorageIdentityCache.cpp | 12 +- .../ObjectStorageIdentityCache.h | 27 +++-- .../ObjectStorages/S3/S3ObjectStorage.cpp | 23 +++- src/IO/ReadBufferFromS3.cpp | 20 +++- src/IO/ReadBufferFromS3.h | 9 +- ..._identity_cache_external_replace.reference | 1 + ...storage_identity_cache_external_replace.sh | 103 ++++++++++++++++++ 7 files changed, 176 insertions(+), 19 deletions(-) create mode 100644 tests/queries/0_stateless/05490_object_storage_identity_cache_external_replace.reference create mode 100755 tests/queries/0_stateless/05490_object_storage_identity_cache_external_replace.sh diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp index 30bedf3855cd4..3e6beb932fc2d 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp @@ -41,17 +41,21 @@ std::optional ObjectStorageIdentityCache::tryGet(const St return std::nullopt; } -UInt64 ObjectStorageIdentityCache::currentEpoch() const +ObjectStorageIdentityCache::Read ObjectStorageIdentityCache::beginFetch() { std::lock_guard lock(epoch_mutex); - return epoch; + return Read{epoch, ++next_fetch_seq}; } -void ObjectStorageIdentityCache::setIfEpoch(const String & key, ObjectStorageIdentity identity, UInt64 expected_epoch) +void ObjectStorageIdentityCache::setIfEpoch(const String & key, ObjectStorageIdentity identity, Read read) { std::lock_guard lock(epoch_mutex); - if (epoch != expected_epoch) + if (epoch != read.epoch) return; + /// `Base::get` takes the cache mutex after `epoch_mutex`, the same order as `set` and `remove`. + if (auto existing = Base::get(key); existing && existing->fetch_seq > read.seq) + return; + identity.fetch_seq = read.seq; Base::set(key, std::make_shared(std::move(identity))); } diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h index 65fb0abc104e1..39e457115f2e4 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h @@ -27,6 +27,9 @@ struct ObjectStorageIdentity /// Cumulative start offset of each multipart part (part i covers [part_offsets[i], part_offsets[i+1])). /// Empty when unknown or the object is a single PUT. std::vector part_offsets; + /// Orders stores of this key. A fetch that started earlier must not overwrite one that started later. + /// Not part of the object's identity. + UInt64 fetch_seq = 0; size_t memoryUsage() const { @@ -67,9 +70,10 @@ inline String makeObjectStorageIdentityCacheKey(std::string_view endpoint, std:: /// /// It is owned by the global Context (see Context::getObjectStorageIdentityCache), sized from server /// settings, and cleared by `SYSTEM DROP OBJECT STORAGE IDENTITY CACHE`. A write, copy, or delete by -/// this server removes the entry and advances the epoch. A metadata fetch reads the epoch before its -/// request and stores its result only if the epoch is unchanged, so a fetch that started before a -/// replacement cannot put the old identity back. The next read of that key requests the metadata again. +/// this server, and a read that finds the object replaced or missing, removes the entry and advances +/// the epoch. A metadata fetch takes `beginFetch` before its request and stores its result only if +/// that epoch is unchanged and no fetch of the same key started later, so an older observation cannot +/// put a stale identity back. The next read of that key requests the metadata again. /// Low-level object-storage code reaches the cache via Context::getGlobalContextInstance() and must /// tolerate a null pointer (no global context yet) by simply issuing the metadata request uncached. class ObjectStorageIdentityCache @@ -83,11 +87,18 @@ class ObjectStorageIdentityCache /// Look up an identity by `makeObjectStorageIdentityCacheKey`. Increments the hit/miss ProfileEvents. std::optional tryGet(const String & key); - /// The current epoch. A fetch captures this before the request and passes it to `setIfEpoch`. - UInt64 currentEpoch() const; + /// Epoch at the start of a metadata fetch, and a sequence that orders that fetch against others. + struct Read + { + UInt64 epoch = 0; + UInt64 seq = 0; + }; + + /// Call before the metadata request and pass the result to `setIfEpoch`. + Read beginFetch(); - /// Store `identity` only when `epoch` is still the epoch `currentEpoch` returned. - void setIfEpoch(const String & key, ObjectStorageIdentity identity, UInt64 epoch); + /// Store `identity` only when `read.epoch` is still current and no later fetch of `key` has stored. + void setIfEpoch(const String & key, ObjectStorageIdentity identity, Read read); /// Advance the epoch and drop the entry. A fetch already in flight will not put the old identity back. void invalidate(const String & key); @@ -101,6 +112,8 @@ class ObjectStorageIdentityCache /// Advanced by every `invalidate` and `clear`. Guarded by `epoch_mutex`. It is never reset, so a /// fetch in flight can never match a later epoch by accident. UInt64 epoch = 0; + /// Advanced by every `beginFetch`. Guarded by `epoch_mutex`. + UInt64 next_fetch_seq = 0; }; using ObjectStorageIdentityCachePtr = std::shared_ptr; diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp index 3065ac805178b..9f33b81be3629 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp @@ -359,6 +359,10 @@ std::unique_ptr S3ObjectStorage::readObject( /// NOLINT } } + /// The buffer can outlive this storage. A GET that finds the object replaced or missing drops the + /// cached identity, so the next read requests the metadata again instead of reusing it. + const String identity_key = makeObjectStorageIdentityCacheKey(uri.endpoint, uri.bucket, object.remote_path); + return std::make_unique( clientForRetryProfile(ObjectStorageControlRequest{ .profile = read_settings.object_storage_retry_profile, @@ -380,7 +384,13 @@ std::unique_ptr S3ObjectStorage::readObject( /// NOLINT (object.bytes_size && object.bytes_size != StoredObject::UnknownSize) ? std::optional(object.bytes_size) : std::nullopt, refresh_callback, std::move(blob_storage_log), - object.etag); + object.etag, + [identity_key]() + { + if (auto global_context = Context::getGlobalContextInstance()) + if (auto cache = global_context->getObjectStorageIdentityCache()) + cache->invalidate(identity_key); + }); } SmallObjectDataWithMetadata S3ObjectStorage::readSmallObjectAndGetObjectMetadata( /// NOLINT @@ -1032,11 +1042,12 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool } } - /// Captured before the request. `setIfEpoch` stores the result only while the epoch is unchanged, - /// so a write, copy, delete, or drop that lands during the request cannot be undone by this fetch. - UInt64 identity_epoch = 0; + /// Captured before the request. `setIfEpoch` stores the result only while this epoch is unchanged + /// and no later fetch of the same key has stored, so a write, copy, delete, drop, or an older + /// observation cannot put a stale identity back. + ObjectStorageIdentityCache::Read identity_read; if (identity_cache) - identity_epoch = identity_cache->currentEpoch(); + identity_read = identity_cache->beginFetch(); auto settings_ptr = s3_settings.get(); @@ -1112,7 +1123,7 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool identity_cache->setIfEpoch( identity_key, ObjectStorageIdentity{result.etag, result.size_bytes, result.is_size_known, result.part_offsets}, - identity_epoch); + identity_read); } return result; diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp index 3062b737b4566..2e4a0f1d7cebe 100644 --- a/src/IO/ReadBufferFromS3.cpp +++ b/src/IO/ReadBufferFromS3.cpp @@ -106,7 +106,8 @@ ReadBufferFromS3::ReadBufferFromS3( std::optional file_size_, const S3CredentialsRefreshCallback & credentials_refresh_callback_, BlobStorageLogWriterPtr blob_storage_log_, - const String & expected_etag_) + const String & expected_etag_, + std::function on_identity_rejected_) : ReadBufferFromFileBase() , client_ptr(std::move(client_ptr_)) , bucket(bucket_) @@ -121,6 +122,7 @@ ReadBufferFromS3::ReadBufferFromS3( , restricted_seek(restricted_seek_) , credentials_refresh_callback(credentials_refresh_callback_) , blob_storage_log(std::move(blob_storage_log_)) + , on_identity_rejected(std::move(on_identity_rejected_)) { file_size = file_size_; } @@ -669,11 +671,14 @@ Aws::S3::Model::GetObjectResult ReadBufferFromS3::sendRequest(size_t attempt, si /// on ETag drift. Skip ?versionId= reads (pinned to an immutable version that expected_etag, /// taken from the current version, would falsely mismatch) and empty response ETags. if (version_id.empty() && !expected_etag.empty() && !response_etag.empty() && response_etag != expected_etag) + { + rejectCachedIdentity(); throw Exception( ErrorCodes::S3_OBJECT_CHANGED_DURING_READ, "S3 object {}/{} was replaced during read (etag changed from {} to {}); " "retry the query, or set s3_validate_etag_on_read=0 to disable this check", bucket, key, expected_etag, response_etag); + } if (blob_storage_log) { @@ -705,15 +710,28 @@ Aws::S3::Model::GetObjectResult ReadBufferFromS3::sendRequest(size_t attempt, si /// error: S3Exception keeps only the Aws S3Errors code, not the HTTP 412 status. if (version_id.empty() && !expected_etag.empty() && error.GetResponseCode() == Aws::Http::HttpResponseCode::PRECONDITION_FAILED) + { + rejectCachedIdentity(); throw Exception( ErrorCodes::S3_OBJECT_CHANGED_DURING_READ, "S3 object {}/{} was replaced during read (If-Match on etag {} failed); " "retry the query, or set s3_validate_etag_on_read=0 to disable this check", bucket, key, expected_etag); + } + + /// A cached size and ETag for a key that is gone would fail the same way on every retry. + if (version_id.empty() && S3::isNotFoundError(error.GetErrorType())) + rejectCachedIdentity(); throw S3Exception(error.GetMessage(), error.GetErrorType()); } +void ReadBufferFromS3::rejectCachedIdentity() const +{ + if (on_identity_rejected) + on_identity_rejected(); +} + ObjectMetadata ReadBufferFromS3::getObjectMetadataFromTheLastRequest() const { if (!impl) diff --git a/src/IO/ReadBufferFromS3.h b/src/IO/ReadBufferFromS3.h index d99c61f015a72..dff4216827b2a 100644 --- a/src/IO/ReadBufferFromS3.h +++ b/src/IO/ReadBufferFromS3.h @@ -65,7 +65,8 @@ class ReadBufferFromS3 : public ReadBufferFromFileBase std::optional file_size = std::nullopt, const S3CredentialsRefreshCallback & credentials_refresh_callback_ = [] {return nullptr;}, BlobStorageLogWriterPtr blob_storage_log_ = {}, - const String & expected_etag_ = {} + const String & expected_etag_ = {}, + std::function on_identity_rejected_ = {} ); ~ReadBufferFromS3() override = default; @@ -125,6 +126,9 @@ class ReadBufferFromS3 : public ReadBufferFromFileBase Aws::S3::Model::GetObjectResult sendRequest(size_t attempt, size_t range_begin, std::optional range_end_incl) const; + /// The cached identity of this key is not the object that was read. Drop it so a retry fetches again. + void rejectCachedIdentity() const; + /// Drops the identity baseline. Called when the next request is a reissue for a range the caller /// explicitly repositioned to (seek, or a change of the read-until bound), as opposed to a retry of /// the same range after a failure: the bytes already delivered before the reposition reached the @@ -159,6 +163,9 @@ class ReadBufferFromS3 : public ReadBufferFromFileBase const S3CredentialsRefreshCallback credentials_refresh_callback; mutable BlobStorageLogWriterPtr blob_storage_log; + + /// Called when a GET finds this key replaced or missing. Empty when the caller has no identity cache. + std::function on_identity_rejected; }; } diff --git a/tests/queries/0_stateless/05490_object_storage_identity_cache_external_replace.reference b/tests/queries/0_stateless/05490_object_storage_identity_cache_external_replace.reference new file mode 100644 index 0000000000000..d86bac9de59ab --- /dev/null +++ b/tests/queries/0_stateless/05490_object_storage_identity_cache_external_replace.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/05490_object_storage_identity_cache_external_replace.sh b/tests/queries/0_stateless/05490_object_storage_identity_cache_external_replace.sh new file mode 100755 index 0000000000000..26f604e5e95b9 --- /dev/null +++ b/tests/queries/0_stateless/05490_object_storage_identity_cache_external_replace.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-parallel-replicas +# Tag no-fasttest: reads a file through the mock S3 server at localhost:11111 +# Tag no-parallel: the identity cache is process-global, and another test may clear it +# Tag no-parallel-replicas: per-query S3 request counts change when a read is spread across replicas + +# A replace that does not go through this server leaves the cached size and ETag in place. +# The next read is pinned to that ETag, fails with S3_OBJECT_CHANGED_DURING_READ, and must +# drop the entry. The retry misses, HEADs, and returns the new rows. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +URL="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" +READ_SETTINGS="max_threads = 1, use_hive_partitioning = 0, parallel_replicas_for_cluster_engines = 0, enable_filesystem_cache = 0, use_page_cache_for_object_storage = 0, s3_validate_etag_on_read = 1, use_object_storage_list_objects_cache = 0" + +fail() +{ + echo "FAIL: $*" >&2 + exit 1 +} + +# $1 label, $2 expected rows (tab-separated query result), $3 head mode, $4 hits mode, $5 misses mode +read_and_check() +{ + local label=$1 + local expected=$2 + local head_mode=$3 + local hits_mode=$4 + local misses_mode=$5 + local qid="idcache_ext_${label}_${CLICKHOUSE_DATABASE}" + local data row head attrs hits misses lists + + data=$(${CLICKHOUSE_CLIENT} --query_id "$qid" -q \ + "SELECT n FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') ORDER BY n SETTINGS ${READ_SETTINGS} FORMAT TabSeparated") \ + || fail "${label}: select failed" + [[ $data == "$expected" ]] || fail "${label}: data '${data}', expected '${expected}'" + + ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "${label}: flush logs failed" + row=$(${CLICKHOUSE_CLIENT} -q " + SELECT + ProfileEvents['S3HeadObject'], + ProfileEvents['S3GetObjectAttributes'], + ProfileEvents['ObjectStorageIdentityCacheHits'], + ProfileEvents['ObjectStorageIdentityCacheMisses'], + ProfileEvents['S3ListObjects'] + FROM system.query_log + WHERE event_date >= yesterday() + AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND query_id = '${qid}' + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC + LIMIT 1 + FORMAT TabSeparated") || fail "${label}: query_log failed" + [[ -n $row ]] || fail "${label}: no query_log row" + + IFS=$'\t' read -r head attrs hits misses lists <<< "$row" + check_count "$label" S3HeadObject "$head" "$head_mode" + check_count "$label" S3GetObjectAttributes "$attrs" eq0 + check_count "$label" Hits "$hits" "$hits_mode" + check_count "$label" Misses "$misses" "$misses_mode" + [[ $lists == "0" ]] || fail "${label}: S3ListObjects=${lists}, expected 0 (row: ${row})" +} + +check_count() +{ + local label=$1 + local name=$2 + local got=$3 + local mode=$4 + case "$mode" in + eq0) [[ $got == "0" ]] || fail "${label}: ${name}=${got}, expected 0" ;; + ge1) [[ $got -ge 1 ]] || fail "${label}: ${name}=${got}, expected >= 1" ;; + *) fail "${label}: bad mode ${mode}" ;; + esac +} + +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear failed" + +${CLICKHOUSE_CLIENT} -q " + INSERT INTO FUNCTION s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') + SELECT 1 + SETTINGS s3_truncate_on_insert = 1" || fail "insert failed" + +read_and_check cold $'1' ge1 eq0 ge1 +read_and_check warm $'1' eq0 ge1 eq0 + +# This PUT does not go through S3ObjectStorage::writeObject, so nothing drops the cached identity. +payload="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" +printf '10\n20\n30\n' > "$payload" +curl -fsS -X PUT "$URL" --upload-file "$payload" >/dev/null || fail "external put failed" + +stale_err=$(${CLICKHOUSE_CLIENT} -q \ + "SELECT n FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') ORDER BY n SETTINGS ${READ_SETTINGS} FORMAT TabSeparated" 2>&1) \ + && fail "stale: the replaced object was read" +[[ $stale_err == *"S3_OBJECT_CHANGED_DURING_READ"* ]] || fail "stale: unexpected error: ${stale_err}" + +read_and_check retried $'10\n20\n30' ge1 eq0 ge1 +read_and_check retried_warm $'10\n20\n30' eq0 ge1 eq0 + +echo OK From 87e09445f3f701aebb76ba549118355558460f5c Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 17 Nov 2026 19:18:11 +0100 Subject: [PATCH 17/37] Object storage: keep the url() shard index when fetching object metadata The metadata fetch for a listed `url()` file called the string overload of `getObjectMetadata`. That overload builds a `RelativePathWithMetadata` without `read_source_index`, so `WebObjectStorage::buildURLs` probed every shard. With several shards that expose the same relative path, the size and ETag could come from a different shard than the one that listed the file, and the read bound could be wrong. `IObjectStorage` gains a `getObjectMetadata` overload that takes the whole `RelativePathWithMetadata` and the part-offset flag. Its default forwards to the object overload, so `WebObjectStorage` keeps the index. `S3ObjectStorage` overrides it to pass the flag to its existing fetch. `createReader` and `createReadBuffer` now call this overload. Test 05491 serves the same file from two local HTTP shards with different sizes and checks that the rows from both shards are returned in full. Co-authored-by: Cursor --- .../ObjectStorages/IObjectStorage.h | 9 ++ .../ObjectStorages/S3/S3ObjectStorage.h | 5 + .../StorageObjectStorageSource.cpp | 4 +- .../05491_url_object_metadata_shard.reference | 1 + .../05491_url_object_metadata_shard.sh | 99 +++++++++++++++++++ 5 files changed, 116 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/05491_url_object_metadata_shard.reference create mode 100755 tests/queries/0_stateless/05491_url_object_metadata_shard.sh diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h b/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h index 96a8357ecdc68..8075e175fd052 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h @@ -373,6 +373,15 @@ class IObjectStorage return getObjectMetadata(path, with_tags); } + /// Same request as the string overload, but the object keeps whatever identifies the replica + /// (a `url()` shard index). The default calls the two-argument object overload so a backend that + /// overrides that one, such as `WebObjectStorage`, still sees the index. Calling the string + /// overload instead would probe every shard and could return another shard's size and ETag. + virtual ObjectMetadata getObjectMetadata(const RelativePathWithMetadata & object, bool with_tags, bool /*fetch_part_offsets*/) const + { + return getObjectMetadata(object, with_tags); + } + /// Same as getObjectMetadata(), but ignores if object does not exist. virtual std::optional tryGetObjectMetadata(const std::string & path, bool with_tags) const = 0; virtual std::optional tryGetObjectMetadata(const RelativePathWithMetadata & object, bool with_tags) const diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h index 2430e659798cc..88ad4c178a5d6 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h @@ -150,6 +150,11 @@ class S3ObjectStorage : public IObjectStorage } ObjectMetadata getObjectMetadata(const std::string & path, bool with_tags, bool fetch_part_offsets) const override; + ObjectMetadata getObjectMetadata(const RelativePathWithMetadata & object, bool with_tags, bool fetch_part_offsets) const override + { + return getObjectMetadata(object.getPath(), with_tags, fetch_part_offsets); + } + std::optional tryGetObjectMetadata(const std::string & path, bool with_tags) const override; /// Marks the HEAD request eligible for the typed NativeConditional mode, so the CAS backend's diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index 95f34405399a4..89ec9fed18877 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -1163,7 +1163,7 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade else { object_info->setObjectMetadata(storage_to_use->getObjectMetadata( - metadata_object.getPath(), with_tags, + metadata_object, with_tags, context_->getSettingsRef()[Setting::object_storage_identity_cache_fetch_part_offsets])); } } @@ -1979,7 +1979,7 @@ std::unique_ptr createReadBuffer( if (!object_info.metadata) { object_info.metadata = object_storage->getObjectMetadata( - object_info.getPath(), /*with_tags=*/ false, + object_info, /*with_tags=*/ false, /*fetch_part_offsets=*/ effective_read_settings.object_storage_identity_cache_fetch_part_offsets); } else if (!object_info.metadata->is_fetched && validate_etag_on_read) diff --git a/tests/queries/0_stateless/05491_url_object_metadata_shard.reference b/tests/queries/0_stateless/05491_url_object_metadata_shard.reference new file mode 100644 index 0000000000000..d86bac9de59ab --- /dev/null +++ b/tests/queries/0_stateless/05491_url_object_metadata_shard.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/05491_url_object_metadata_shard.sh b/tests/queries/0_stateless/05491_url_object_metadata_shard.sh new file mode 100755 index 0000000000000..889c23190766f --- /dev/null +++ b/tests/queries/0_stateless/05491_url_object_metadata_shard.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: serves two local HTTP shards so url() can list the same path on each. + +# Two url() shards expose the same relative path with different sizes. Metadata for each listed +# file must be read from that file's shard. Losing the shard index HEADs the first shard for both +# and uses its size as the read bound, so the larger file is truncated. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +fail() +{ + echo "FAIL: $*" >&2 + exit 1 +} + +free_port() +{ + python3 -c " +import socket +s = socket.socket() +s.bind(('127.0.0.1', 0)) +print(s.getsockname()[1]) +s.close() +" +} + +PORT_A=$(free_port) +PORT_B=$(free_port) +[[ -n $PORT_A && -n $PORT_B && $PORT_A != "$PORT_B" ]] || fail "could not reserve two ports" + +python3 -c " +from http.server import HTTPServer, BaseHTTPRequestHandler +import threading + +SMALL = b'1\n' +LARGE = b'10\n20\n30\n' +INDEX = b'file.tsv\n' + +def make_handler(body): + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def respond(self, with_body): + path = self.path.split('?')[0] + if path == '/data/': + data, content_type = INDEX, 'text/html' + elif path == '/data/file.tsv': + data, content_type = body, 'text/plain' + else: + self.send_response(404) + self.end_headers() + return + self.send_response(200) + self.send_header('Content-Type', content_type) + self.send_header('Content-Length', str(len(data))) + self.end_headers() + if with_body: + self.wfile.write(data) + + def do_HEAD(self): + self.respond(False) + + def do_GET(self): + self.respond(True) + return Handler + +def serve(port, body): + HTTPServer(('127.0.0.1', port), make_handler(body)).serve_forever() + +threading.Thread(target=serve, args=(${PORT_A}, SMALL), daemon=True).start() +serve(${PORT_B}, LARGE) +" & +HTTP_PID=$! +trap 'kill $HTTP_PID 2>/dev/null; wait $HTTP_PID 2>/dev/null' EXIT + +for _ in $(seq 1 50); do + curl -sf "http://127.0.0.1:${PORT_A}/data/" >/dev/null && curl -sf "http://127.0.0.1:${PORT_B}/data/" >/dev/null && break + sleep 0.1 +done +curl -sf "http://127.0.0.1:${PORT_A}/data/" >/dev/null || fail "shard A did not start" +curl -sf "http://127.0.0.1:${PORT_B}/data/" >/dev/null || fail "shard B did not start" + +URL="http://127.0.0.1:{${PORT_A},${PORT_B}}/data/*.tsv" +data=$(${CLICKHOUSE_CLIENT} -q " + SET allow_experimental_url_wildcard_from_index_pages = 1; + SELECT n + FROM url('${URL}', 'TSV', 'n UInt64') + ORDER BY n + SETTINGS max_threads = 1, http_max_tries = 1 + FORMAT TabSeparated") || fail "select failed" + +expected=$'1\n10\n20\n30' +[[ $data == "$expected" ]] || fail "data '${data}', expected '${expected}'" + +echo OK From e1cf3b485ff4105293394ce84842bf56489948bd Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 17 Nov 2026 19:22:16 +0100 Subject: [PATCH 18/37] Object storage: return Last-Modified from the identity cache A cache hit in `S3ObjectStorage::getObjectMetadata` returned the object with `is_last_modified_known = false`, so the `_time` virtual column became NULL on every read after the first. The identity cache did not store Last-Modified, and leaving the epoch with the flag set would have made the schema and count caches treat the object as older than every cached entry. `ObjectStorageIdentity` now stores the Last-Modified value and whether it is known. The miss path records the HEAD `last_modification_time`. A hit returns the stored time with the flag set. An unknown time still produces an unknown result on hit, so the schema and count caches keep treating the epoch as a missing timestamp. The new field is placed after `fetch_seq`, so the positional initializer for the identity keeps its meaning. Added the test `05492_object_storage_identity_cache_last_modified`, which reads one S3 object twice and requires the same non-NULL `_time` with a cache hit and no HEAD on the second read. Found by the audit of the identity cache PR: https://github.com/Altinity/ClickHouse/pull/2888 Co-authored-by: Cursor --- .../ObjectStorageIdentityCache.h | 5 +- .../ObjectStorages/S3/S3ObjectStorage.cpp | 22 +++-- ...age_identity_cache_last_modified.reference | 1 + ...ct_storage_identity_cache_last_modified.sh | 87 +++++++++++++++++++ 4 files changed, 106 insertions(+), 9 deletions(-) create mode 100644 tests/queries/0_stateless/05492_object_storage_identity_cache_last_modified.reference create mode 100755 tests/queries/0_stateless/05492_object_storage_identity_cache_last_modified.sh diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h index 39e457115f2e4..0bd23abe02cc7 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h @@ -12,7 +12,7 @@ namespace DB { /// Object-storage "identity": the bits needed to open an object without a fresh HEAD - its size, -/// ETag, and (if multipart-uploaded) the byte offsets of its parts. +/// ETag, Last-Modified, and (if multipart-uploaded) the byte offsets of its parts. /// /// It exists to break the ordering that forces a HEAD before every open: object caches key on /// (path, etag), so the etag must be known up front. Here the key is the store plus the path @@ -27,6 +27,9 @@ struct ObjectStorageIdentity /// Cumulative start offset of each multipart part (part i covers [part_offsets[i], part_offsets[i+1])). /// Empty when unknown or the object is a single PUT. std::vector part_offsets; + /// Seconds since epoch. Meaningful only when `is_last_modified_known`. + Int64 last_modified = 0; + bool is_last_modified_known = false; /// Orders stores of this key. A fetch that started earlier must not overwrite one that started later. /// Not part of the object's identity. UInt64 fetch_seq = 0; diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp index 9f33b81be3629..efaef18ad5dd7 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -1033,10 +1034,10 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool result.is_size_known = cached->is_size_known; result.etag = cached->etag; result.part_offsets = cached->part_offsets; - /// The cache does not store Last-Modified. Leaving it at the epoch with - /// `is_last_modified_known == true` would make schema and count caches treat the object - /// as older than every cached entry. - result.is_last_modified_known = false; + if (cached->is_last_modified_known) + result.last_modified = Poco::Timestamp::fromEpochTime(static_cast(cached->last_modified)); + else + result.is_last_modified_known = false; return result; } } @@ -1120,10 +1121,15 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool /// The read that reached this point already holds the identity from before any overwrite that /// happened while it was paused. Resuming must not store that identity. FailPointInjection::pauseFailPoint(FailPoints::object_storage_identity_cache_pause_before_store); - identity_cache->setIfEpoch( - identity_key, - ObjectStorageIdentity{result.etag, result.size_bytes, result.is_size_known, result.part_offsets}, - identity_read); + ObjectStorageIdentity identity; + identity.etag = result.etag; + identity.size = result.size_bytes; + identity.is_size_known = result.is_size_known; + identity.part_offsets = result.part_offsets; + identity.is_last_modified_known = result.is_last_modified_known; + if (result.is_last_modified_known) + identity.last_modified = static_cast(result.last_modified.epochTime()); + identity_cache->setIfEpoch(identity_key, std::move(identity), identity_read); } return result; diff --git a/tests/queries/0_stateless/05492_object_storage_identity_cache_last_modified.reference b/tests/queries/0_stateless/05492_object_storage_identity_cache_last_modified.reference new file mode 100644 index 0000000000000..d86bac9de59ab --- /dev/null +++ b/tests/queries/0_stateless/05492_object_storage_identity_cache_last_modified.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/05492_object_storage_identity_cache_last_modified.sh b/tests/queries/0_stateless/05492_object_storage_identity_cache_last_modified.sh new file mode 100755 index 0000000000000..2ded1d045ca58 --- /dev/null +++ b/tests/queries/0_stateless/05492_object_storage_identity_cache_last_modified.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-parallel-replicas +# Tag no-fasttest: reads a file through the mock S3 server at localhost:11111 +# Tag no-parallel: the identity cache is process-global, and another test may clear it +# Tag no-parallel-replicas: per-query S3 request counts change when a read is spread across replicas + +# A cache hit must return the Last-Modified stored with the identity. Otherwise the second read of +# the same object reports `_time` as NULL. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +URL="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" +READ_SETTINGS="max_threads = 1, use_hive_partitioning = 0, parallel_replicas_for_cluster_engines = 0, enable_filesystem_cache = 0, use_page_cache_for_object_storage = 0, s3_validate_etag_on_read = 0, use_object_storage_list_objects_cache = 0" + +fail() +{ + echo "FAIL: $*" >&2 + exit 1 +} + +# $1 label, $2 hits mode, $3 misses mode. Prints the _time value. +read_time() +{ + local label=$1 + local hits_mode=$2 + local misses_mode=$3 + local qid="idcache_time_${label}_${CLICKHOUSE_DATABASE}" + local value row head hits misses + + value=$(${CLICKHOUSE_CLIENT} --query_id "$qid" -q " + SELECT if(isNull(_time), 'NULL', toString(_time)) + FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') + SETTINGS ${READ_SETTINGS} + FORMAT TabSeparated") || fail "${label}: select failed" + + ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "${label}: flush logs failed" + row=$(${CLICKHOUSE_CLIENT} -q " + SELECT + ProfileEvents['S3HeadObject'], + ProfileEvents['ObjectStorageIdentityCacheHits'], + ProfileEvents['ObjectStorageIdentityCacheMisses'] + FROM system.query_log + WHERE event_date >= yesterday() + AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND query_id = '${qid}' + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC + LIMIT 1 + FORMAT TabSeparated") || fail "${label}: query_log failed" + [[ -n $row ]] || fail "${label}: no query_log row" + + IFS=$'\t' read -r head hits misses <<< "$row" + case "$hits_mode" in + eq0) [[ $hits == "0" ]] || fail "${label}: Hits=${hits}, expected 0" ;; + ge1) + [[ $hits -ge 1 ]] || fail "${label}: Hits=${hits}, expected >= 1" + [[ $head == "0" ]] || fail "${label}: cache hit still sent S3HeadObject=${head}" + ;; + esac + case "$misses_mode" in + eq0) [[ $misses == "0" ]] || fail "${label}: Misses=${misses}, expected 0" ;; + ge1) + [[ $misses -ge 1 ]] || fail "${label}: Misses=${misses}, expected >= 1" + [[ $head -ge 1 ]] || fail "${label}: S3HeadObject=${head}, expected >= 1" + ;; + esac + + printf '%s' "$value" +} + +${CLICKHOUSE_CLIENT} -q " + INSERT INTO FUNCTION s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') + SELECT 1 + SETTINGS s3_truncate_on_insert = 1" || fail "insert failed" + +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear failed" + +cold=$(read_time cold eq0 ge1) +warm=$(read_time warm ge1 eq0) + +[[ $cold != "NULL" && -n $cold ]] || fail "cold _time is '${cold}'" +[[ $warm == "$cold" ]] || fail "warm _time '${warm}', cold '${cold}'" + +echo OK From 4309cb868027a73c99f6e02d56c3e51cdc5495f4 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 17 Nov 2026 19:36:49 +0100 Subject: [PATCH 19/37] Object storage: keep a cached Last-Modified out of schema and count freshness An identity-cache hit now returns the stored Last-Modified, so `_time` is no longer NULL on repeat reads. The same value was also consumed by the schema cache (`ReadBufferIterator`) and the count-from-files cache (`StorageObjectStorageSource`). After an external replace, the stored time is older than the registration time of the cached schema and row count, so both caches would treat the replaced object as unchanged and return the previous result. `ObjectMetadata` gets `is_last_modified_fresh`, which is cleared when the time came from the identity cache, and `lastModifiedConfirmsUnchanged()`. Both caches now require that check instead of only `is_last_modified_known`. A HEAD or listing that just observed the time still allows the caches to use it. A read pinned to the cached ETag still fails with `S3_OBJECT_CHANGED_DURING_READ` after a replace, and the retry loads the new object, so the count is correct after one failed query. Added the test `05493_object_storage_identity_cache_count_after_replace`: it reads `count()` once, replaces the object with a PUT that bypasses the server, expects `S3_OBJECT_CHANGED_DURING_READ`, and then expects the new count. Found by the audit of the identity cache PR: https://github.com/Altinity/ClickHouse/pull/2888 Co-authored-by: Cursor --- .../ObjectStorages/IObjectStorage.h | 10 ++++ .../ObjectStorages/S3/S3ObjectStorage.cpp | 5 ++ .../ObjectStorage/ReadBufferIterator.cpp | 5 +- .../StorageObjectStorageSource.cpp | 7 +-- ...entity_cache_count_after_replace.reference | 1 + ...rage_identity_cache_count_after_replace.sh | 53 +++++++++++++++++++ 6 files changed, 76 insertions(+), 5 deletions(-) create mode 100644 tests/queries/0_stateless/05493_object_storage_identity_cache_count_after_replace.reference create mode 100755 tests/queries/0_stateless/05493_object_storage_identity_cache_count_after_replace.sh diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h b/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h index 8075e175fd052..a0920e3ae38a7 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h @@ -139,6 +139,16 @@ struct ObjectMetadata /// default epoch would make the schema/count caches treat the object as older than any cached entry and /// silently reuse a stale value. Cache validators must skip the cache when this is `false`. bool is_last_modified_known = true; + /// False when `last_modified` was reused from the identity cache. `_time` may still show it, but it was + /// not observed by this request. An external replace leaves that stored time older than the schema and + /// count registration time, so those caches would keep the previous schema and row count. + bool is_last_modified_fresh = true; + + /// Schema and count caches may use `last_modified` only when this request observed it. + bool lastModifiedConfirmsUnchanged() const + { + return is_last_modified_known && is_last_modified_fresh; + } std::string etag; /// Whether `etag` is a strong content/version identifier, i.e. different content is /// guaranteed to produce a different `etag`. Real S3/Azure ETags and the sub-second diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp index efaef18ad5dd7..e49c883b92bbb 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp @@ -1035,7 +1035,12 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool result.etag = cached->etag; result.part_offsets = cached->part_offsets; if (cached->is_last_modified_known) + { result.last_modified = Poco::Timestamp::fromEpochTime(static_cast(cached->last_modified)); + /// Real enough for `_time`, but not a freshness proof. The schema and count caches + /// treat an older time as "unchanged" and would keep a replaced object's schema and row count. + result.is_last_modified_fresh = false; + } else result.is_last_modified_known = false; return result; diff --git a/src/Storages/ObjectStorage/ReadBufferIterator.cpp b/src/Storages/ObjectStorage/ReadBufferIterator.cpp index 8874a4be95837..09e01becdeb0f 100644 --- a/src/Storages/ObjectStorage/ReadBufferIterator.cpp +++ b/src/Storages/ObjectStorage/ReadBufferIterator.cpp @@ -93,8 +93,9 @@ std::optional ReadBufferIterator::tryGetColumnsFromCache( /// An unknown modification time (e.g. a web object whose HTTP response has no `Last-Modified` /// header) must be reported as unavailable, not as the epoch: the schema cache reuses a cached /// entry whenever the modification time is older than the registration time, so a fake epoch-`0` - /// would keep a stale schema valid forever. Returning `nullopt` makes the cache skip the entry. - if (!metadata || !metadata->is_last_modified_known) + /// would keep a stale schema valid forever. A time reused from the identity cache has the same + /// effect after an external replace. Returning `nullopt` makes the cache skip the entry. + if (!metadata || !metadata->lastModifiedConfirmsUnchanged()) return std::nullopt; return std::optional(metadata->last_modified.epochTime()); }; diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index 89ec9fed18877..59b3b9429bccf 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -1242,9 +1242,10 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade { const auto metadata = object_info->getObjectMetadata(); /// An unknown modification time (e.g. a web object without a `Last-Modified` header) must not be - /// reported as the epoch, otherwise the stale cached row count would always look valid. Reporting - /// it as unavailable makes the count cache re-read the file instead. - if (!metadata || !metadata->is_last_modified_known) + /// reported as the epoch, otherwise the stale cached row count would always look valid. A time + /// reused from the identity cache is also not proof the object is unchanged. Reporting either + /// as unavailable makes the count cache re-read the file instead. + if (!metadata || !metadata->lastModifiedConfirmsUnchanged()) return std::nullopt; return std::optional(metadata->last_modified.epochTime()); }; diff --git a/tests/queries/0_stateless/05493_object_storage_identity_cache_count_after_replace.reference b/tests/queries/0_stateless/05493_object_storage_identity_cache_count_after_replace.reference new file mode 100644 index 0000000000000..d86bac9de59ab --- /dev/null +++ b/tests/queries/0_stateless/05493_object_storage_identity_cache_count_after_replace.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/05493_object_storage_identity_cache_count_after_replace.sh b/tests/queries/0_stateless/05493_object_storage_identity_cache_count_after_replace.sh new file mode 100755 index 0000000000000..1353102740abe --- /dev/null +++ b/tests/queries/0_stateless/05493_object_storage_identity_cache_count_after_replace.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-parallel-replicas +# Tag no-fasttest: reads a file through the mock S3 server at localhost:11111 +# Tag no-parallel: the identity cache and the schema cache are process-global +# Tag no-parallel-replicas: per-query S3 request counts change when a read is spread across replicas + +# An identity-cache hit returns the stored Last-Modified for `_time`, but that time must not prove +# the object is unchanged. Otherwise `count()` keeps the previous row count after an external replace. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +URL="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" +READ_SETTINGS="max_threads = 1, use_hive_partitioning = 0, parallel_replicas_for_cluster_engines = 0, enable_filesystem_cache = 0, use_page_cache_for_object_storage = 0, s3_validate_etag_on_read = 1, use_object_storage_list_objects_cache = 0, use_cache_for_count_from_files = 1, schema_inference_use_cache_for_s3 = 1" + +fail() +{ + echo "FAIL: $*" >&2 + exit 1 +} + +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear failed" + +${CLICKHOUSE_CLIENT} -q " + INSERT INTO FUNCTION s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') + SELECT 1 + SETTINGS s3_truncate_on_insert = 1" || fail "insert failed" + +# Last-Modified has second resolution. The count is registered in the same second as the upload unless +# we wait, and a time equal to the registration does not reproduce the stale-count hit. +sleep 1 + +count1=$(${CLICKHOUSE_CLIENT} -q " + SELECT count() FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') + SETTINGS ${READ_SETTINGS}") || fail "first count failed" +[[ $count1 == "1" ]] || fail "first count '${count1}', expected 1" + +payload="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" +printf '10\n20\n30\n' > "$payload" +curl -fsS -X PUT "$URL" --upload-file "$payload" >/dev/null || fail "external put failed" + +stale_err=$(${CLICKHOUSE_CLIENT} -q " + SELECT count() FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') + SETTINGS ${READ_SETTINGS}" 2>&1) && fail "replaced object: count succeeded with '${stale_err}'" +[[ $stale_err == *"S3_OBJECT_CHANGED_DURING_READ"* ]] || fail "replaced object: unexpected error: ${stale_err}" + +count2=$(${CLICKHOUSE_CLIENT} -q " + SELECT count() FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') + SETTINGS ${READ_SETTINGS}") || fail "retry count failed" +[[ $count2 == "3" ]] || fail "retry count '${count2}', expected 3" + +echo OK From a5016ad7180e1da5dea05b107240098b69e4e1a4 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 17 Nov 2026 19:41:31 +0100 Subject: [PATCH 20/37] Object storage: revalidate the identity before a filesystem-cache hit A filesystem cache (and the page cache) is keyed by the object's ETag. When the identity cache supplies that ETag, a fully cached read never issues a GET, so the `If-Match` check in `ReadBufferFromS3` never runs. After an external replace the cache keeps serving the previous bytes under the old ETag, and nothing in the query reports the change. `ObjectMetadata` gets `is_from_identity_cache`, set on an identity-cache hit. `createReadBuffer` checks it when a filesystem or page cache is about to use the ETag as the key, and re-reads the object through the new `getObjectMetadataIgnoringIdentityCache`. That skips the lookup but still stores the result, so the next read is cached again. An unchanged object keeps its ETag and is served from the cache. A replaced object gets a new key and is read from object storage. The added cost is one HEAD per filesystem-cache read whose identity came from the identity cache. A cold read is unchanged. Added the test `05494_object_storage_identity_cache_filesystem_cache_replace`: it reads an object into `cache_for_readbigat`, checks that the next read is a cache hit that still sends a HEAD, replaces the object with a PUT that bypasses the server, and expects the new rows. Found by the audit of the identity cache PR: https://github.com/Altinity/ClickHouse/pull/2888 Co-authored-by: Cursor --- .../ObjectStorages/IObjectStorage.h | 11 +++ .../ObjectStorages/S3/S3ObjectStorage.cpp | 5 +- .../ObjectStorages/S3/S3ObjectStorage.h | 18 +++- .../StorageObjectStorageSource.cpp | 14 +++ ...y_cache_filesystem_cache_replace.reference | 1 + ...identity_cache_filesystem_cache_replace.sh | 93 +++++++++++++++++++ 6 files changed, 137 insertions(+), 5 deletions(-) create mode 100644 tests/queries/0_stateless/05494_object_storage_identity_cache_filesystem_cache_replace.reference create mode 100755 tests/queries/0_stateless/05494_object_storage_identity_cache_filesystem_cache_replace.sh diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h b/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h index a0920e3ae38a7..ccd6cdc1fa943 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h @@ -143,6 +143,9 @@ struct ObjectMetadata /// not observed by this request. An external replace leaves that stored time older than the schema and /// count registration time, so those caches would keep the previous schema and row count. bool is_last_modified_fresh = true; + /// True when size and ETag were reused from the identity cache rather than observed by this request. + /// A filesystem or page cache keyed by that ETag can then serve the previous bytes without a GET. + bool is_from_identity_cache = false; /// Schema and count caches may use `last_modified` only when this request observed it. bool lastModifiedConfirmsUnchanged() const @@ -392,6 +395,14 @@ class IObjectStorage return getObjectMetadata(object, with_tags); } + /// Same as `getObjectMetadata`, but do not answer from the identity cache. A content cache (filesystem + /// cache, page cache) keyed by a cached ETag would otherwise serve a replaced object's old bytes, because + /// a full hit never issues the GET that compares ETags. The default has no identity cache. + virtual ObjectMetadata getObjectMetadataIgnoringIdentityCache(const RelativePathWithMetadata & object, bool with_tags, bool fetch_part_offsets) const + { + return getObjectMetadata(object, with_tags, fetch_part_offsets); + } + /// Same as getObjectMetadata(), but ignores if object does not exist. virtual std::optional tryGetObjectMetadata(const std::string & path, bool with_tags) const = 0; virtual std::optional tryGetObjectMetadata(const RelativePathWithMetadata & object, bool with_tags) const diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp index e49c883b92bbb..3026635d8164e 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp @@ -1003,7 +1003,7 @@ std::optional S3ObjectStorage::tryGetObjectMetadataImpl( return result; } -ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool with_tags, bool fetch_part_offsets) const +ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool with_tags, bool fetch_part_offsets, bool use_identity_cache) const { /// When the caller only needs identity (size / etag / part offsets, not tags or user metadata) we /// take a fast path: consult the identity cache to skip the metadata request entirely on repeat @@ -1025,7 +1025,7 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool if (auto global_context = Context::getGlobalContextInstance()) identity_cache = global_context->getObjectStorageIdentityCache(); - if (identity_cache) + if (use_identity_cache && identity_cache) { if (auto cached = identity_cache->tryGet(identity_key)) { @@ -1034,6 +1034,7 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool result.is_size_known = cached->is_size_known; result.etag = cached->etag; result.part_offsets = cached->part_offsets; + result.is_from_identity_cache = true; if (cached->is_last_modified_known) { result.last_modified = Poco::Timestamp::fromEpochTime(static_cast(cached->last_modified)); diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h index 88ad4c178a5d6..bd96c35946cc4 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h @@ -146,13 +146,21 @@ class S3ObjectStorage : public IObjectStorage ObjectMetadata getObjectMetadata(const std::string & path, bool with_tags) const override { - return getObjectMetadata(path, with_tags, /*fetch_part_offsets=*/ false); + return getObjectMetadata(path, with_tags, /*fetch_part_offsets=*/ false, /*use_identity_cache=*/ true); + } + ObjectMetadata getObjectMetadata(const std::string & path, bool with_tags, bool fetch_part_offsets) const override + { + return getObjectMetadata(path, with_tags, fetch_part_offsets, /*use_identity_cache=*/ true); } - ObjectMetadata getObjectMetadata(const std::string & path, bool with_tags, bool fetch_part_offsets) const override; ObjectMetadata getObjectMetadata(const RelativePathWithMetadata & object, bool with_tags, bool fetch_part_offsets) const override { - return getObjectMetadata(object.getPath(), with_tags, fetch_part_offsets); + return getObjectMetadata(object.getPath(), with_tags, fetch_part_offsets, /*use_identity_cache=*/ true); + } + + ObjectMetadata getObjectMetadataIgnoringIdentityCache(const RelativePathWithMetadata & object, bool with_tags, bool fetch_part_offsets) const override + { + return getObjectMetadata(object.getPath(), with_tags, fetch_part_offsets, /*use_identity_cache=*/ false); } std::optional tryGetObjectMetadata(const std::string & path, bool with_tags) const override; @@ -257,6 +265,10 @@ class S3ObjectStorage : public IObjectStorage /// Drop the identity-cache entry for `path`. The next read of this key requests the metadata again. void invalidateCachedIdentity(const std::string & path) const; + /// `use_identity_cache` false still stores the result. It only skips the lookup, so a content cache + /// can key itself on an ETag observed by this request. + ObjectMetadata getObjectMetadata(const std::string & path, bool with_tags, bool fetch_part_offsets, bool use_identity_cache) const; + /// Runs `fn` and, if it failed because the vended credentials expired, refreshes this disk's /// client and runs it once more. `fn` must re-read the client itself, so the second run signs /// with the refreshed one. diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index 59b3b9429bccf..8b452e150b2ce 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -1991,6 +1991,20 @@ std::unique_ptr createReadBuffer( object_info.metadata = object_storage->getObjectMetadata(object_info, /*with_tags=*/ false); } + /// A filesystem or page cache hit never issues the GET, so it cannot notice that the object was + /// replaced. Re-read the identity before using its ETag as the cache key. An unchanged object keeps + /// the same key and is served from the cache. A replaced object gets a new key and is read again. + if ((use_filesystem_cache || use_page_cache) + && object_info.metadata + && object_info.metadata->is_from_identity_cache + && object_info.metadata->is_size_known + && object_info.metadata->isEtagUsableAsCacheKey()) + { + object_info.metadata = object_storage->getObjectMetadataIgnoringIdentityCache( + object_info, /*with_tags=*/ false, + /*fetch_part_offsets=*/ effective_read_settings.object_storage_identity_cache_fetch_part_offsets); + } + if (use_page_cache && !object_info.metadata->isEtagUsableAsCacheKey()) { LOG_WARNING(log, "Cannot use page cache, etag is missing or not a strong content identifier"); diff --git a/tests/queries/0_stateless/05494_object_storage_identity_cache_filesystem_cache_replace.reference b/tests/queries/0_stateless/05494_object_storage_identity_cache_filesystem_cache_replace.reference new file mode 100644 index 0000000000000..d86bac9de59ab --- /dev/null +++ b/tests/queries/0_stateless/05494_object_storage_identity_cache_filesystem_cache_replace.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/05494_object_storage_identity_cache_filesystem_cache_replace.sh b/tests/queries/0_stateless/05494_object_storage_identity_cache_filesystem_cache_replace.sh new file mode 100755 index 0000000000000..7383e29a0cd9c --- /dev/null +++ b/tests/queries/0_stateless/05494_object_storage_identity_cache_filesystem_cache_replace.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-parallel-replicas +# Tag no-fasttest: reads a file through the mock S3 server at localhost:11111 +# Tag no-parallel: the identity cache and the filesystem cache are process-global +# Tag no-parallel-replicas: per-query S3 request counts change when a read is spread across replicas + +# A filesystem-cache hit does not issue a GET, so it cannot see that an object was replaced. The read +# must revalidate the identity before using its ETag as the cache key. An unchanged object is still +# served from the cache. A replaced object is read again. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +URL="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" +READ_SETTINGS="max_threads = 1, use_hive_partitioning = 0, parallel_replicas_for_cluster_engines = 0, enable_filesystem_cache = 1, filesystem_cache_name = 'cache_for_readbigat', use_page_cache_for_object_storage = 0, s3_validate_etag_on_read = 1, use_object_storage_list_objects_cache = 0" + +fail() +{ + echo "FAIL: $*" >&2 + exit 1 +} + +# $1 label, $2 expected rows, $3 head mode, $4 get mode, $5 cache-read mode +read_and_check() +{ + local label=$1 + local expected=$2 + local head_mode=$3 + local get_mode=$4 + local cache_mode=$5 + local qid="idcache_fscache_${label}_${CLICKHOUSE_DATABASE}" + local data row head gets cache_bytes + + data=$(${CLICKHOUSE_CLIENT} --query_id "$qid" -q \ + "SELECT n FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') ORDER BY n SETTINGS ${READ_SETTINGS} FORMAT TabSeparated") \ + || fail "${label}: select failed" + [[ $data == "$expected" ]] || fail "${label}: data '${data}', expected '${expected}'" + + ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "${label}: flush logs failed" + row=$(${CLICKHOUSE_CLIENT} -q " + SELECT + ProfileEvents['S3HeadObject'], + ProfileEvents['S3GetObject'], + ProfileEvents['CachedReadBufferReadFromCacheBytes'] + FROM system.query_log + WHERE event_date >= yesterday() + AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND query_id = '${qid}' + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC + LIMIT 1 + FORMAT TabSeparated") || fail "${label}: query_log failed" + [[ -n $row ]] || fail "${label}: no query_log row" + + IFS=$'\t' read -r head gets cache_bytes <<< "$row" + case "$head_mode" in + ge1) [[ $head -ge 1 ]] || fail "${label}: S3HeadObject=${head}, expected >= 1" ;; + *) fail "${label}: bad head mode ${head_mode}" ;; + esac + case "$get_mode" in + eq0) [[ $gets == "0" ]] || fail "${label}: S3GetObject=${gets}, expected 0" ;; + ge1) [[ $gets -ge 1 ]] || fail "${label}: S3GetObject=${gets}, expected >= 1" ;; + *) fail "${label}: bad get mode ${get_mode}" ;; + esac + case "$cache_mode" in + ge1) [[ $cache_bytes -ge 1 ]] || fail "${label}: CachedReadBufferReadFromCacheBytes=${cache_bytes}, expected >= 1" ;; + any) ;; + *) fail "${label}: bad cache mode ${cache_mode}" ;; + esac +} + +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear identity cache failed" +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR FILESYSTEM CACHE 'cache_for_readbigat'" || fail "clear filesystem cache failed" + +${CLICKHOUSE_CLIENT} -q " + INSERT INTO FUNCTION s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') + SELECT 1 + SETTINGS s3_truncate_on_insert = 1" || fail "insert failed" + +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear identity cache after insert failed" + +read_and_check cold $'1' ge1 ge1 any +read_and_check cached $'1' ge1 eq0 ge1 + +payload="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" +printf '10\n20\n30\n' > "$payload" +curl -fsS -X PUT "$URL" --upload-file "$payload" >/dev/null || fail "external put failed" + +read_and_check replaced $'10\n20\n30' ge1 ge1 any + +echo OK From d6e00589944bf503bf53f54bed600de4e3641d20 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 17 Nov 2026 19:46:22 +0100 Subject: [PATCH 21/37] Object storage: do not cache "time unknown" from GetObjectAttributes `GetObjectAttributes` cannot request Last-Modified in this SDK, so a successful part-offset fetch left `last_modification_time` at 0. The identity cache then stored that as "the object has no modification time", and every later `_time` read from the cache was NULL, including reads with part offsets disabled. `S3::ObjectInfo` gets `last_modification_time_observed`, set only by a HEAD response. The identity gets `last_modified_observed`. A cache entry without an observed time is not returned as a complete identity. The read keeps the cached part offsets and issues a HEAD for the time. The stored entry then carries the HEAD's time. The part offsets are kept only while the ETag is unchanged. A HEAD that really returns a zero modification time is still cached as unknown. Added the failpoint `object_storage_identity_cache_skip_last_modified`, which simulates an attributes response without Last-Modified, and the test `05495_object_storage_identity_cache_part_offsets_time`. It reads one object with part offsets enabled and expects `_time` to be NULL. It then disables the failpoint and expects a non-NULL `_time` with a HEAD. Found by the audit of the identity cache PR: https://github.com/Altinity/ClickHouse/pull/2888 Co-authored-by: Cursor --- src/Common/FailPoint.cpp | 1 + .../ObjectStorageIdentityCache.h | 3 + .../ObjectStorages/S3/S3ObjectStorage.cpp | 68 ++++++++++---- src/IO/S3/getObjectInfo.cpp | 1 + src/IO/S3/getObjectInfo.h | 4 + ...identity_cache_part_offsets_time.reference | 1 + ...torage_identity_cache_part_offsets_time.sh | 89 +++++++++++++++++++ 7 files changed, 149 insertions(+), 18 deletions(-) create mode 100644 tests/queries/0_stateless/05495_object_storage_identity_cache_part_offsets_time.reference create mode 100755 tests/queries/0_stateless/05495_object_storage_identity_cache_part_offsets_time.sh diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp index c7c49cfe8c718..1d13be3451a9f 100644 --- a/src/Common/FailPoint.cpp +++ b/src/Common/FailPoint.cpp @@ -85,6 +85,7 @@ static struct InitFiu ONCE(s3_send_request_throw_expired_token) \ REGULAR(s3_read_inject_etag_mismatch) \ PAUSEABLE_ONCE(object_storage_identity_cache_pause_before_store) \ + REGULAR(object_storage_identity_cache_skip_last_modified) \ REGULAR(azure_inject_forbidden_response) \ ONCE(azure_inject_forbidden_response_once) \ REGULAR(azure_inject_auth_failure_on_request) \ diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h index 0bd23abe02cc7..b67f668848dc2 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h @@ -30,6 +30,9 @@ struct ObjectStorageIdentity /// Seconds since epoch. Meaningful only when `is_last_modified_known`. Int64 last_modified = 0; bool is_last_modified_known = false; + /// False when this entry was stored from `GetObjectAttributes`, which does not return Last-Modified. + /// That is not the same as observing that the object has no modification time. + bool last_modified_observed = false; /// Orders stores of this key. A fetch that started earlier must not overwrite one that started later. /// Not part of the object's identity. UInt64 fetch_seq = 0; diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp index 3026635d8164e..ef3b612567a21 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp @@ -64,6 +64,7 @@ namespace DB::FailPoints { extern const char object_storage_force_refresh_callback_success[]; extern const char object_storage_identity_cache_pause_before_store[]; + extern const char object_storage_identity_cache_skip_last_modified[]; } @@ -1018,6 +1019,11 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool const bool identity_only = !with_tags; /// Bucket and path alone collide when two endpoints reuse a bucket name. const String identity_key = makeObjectStorageIdentityCacheKey(uri.endpoint, uri.bucket, path); + /// A `GetObjectAttributes` entry has size, ETag and part offsets, but this SDK cannot ask it for + /// Last-Modified. Keep those offsets and HEAD for the time instead of caching "time unknown". + bool fill_last_modified = false; + String preserved_etag; + std::vector preserved_part_offsets; ObjectStorageIdentityCachePtr identity_cache; if (identity_only) @@ -1029,22 +1035,29 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool { if (auto cached = identity_cache->tryGet(identity_key)) { - ObjectMetadata result; - result.size_bytes = cached->size; - result.is_size_known = cached->is_size_known; - result.etag = cached->etag; - result.part_offsets = cached->part_offsets; - result.is_from_identity_cache = true; - if (cached->is_last_modified_known) + if (cached->last_modified_observed) { - result.last_modified = Poco::Timestamp::fromEpochTime(static_cast(cached->last_modified)); - /// Real enough for `_time`, but not a freshness proof. The schema and count caches - /// treat an older time as "unchanged" and would keep a replaced object's schema and row count. - result.is_last_modified_fresh = false; + ObjectMetadata result; + result.size_bytes = cached->size; + result.is_size_known = cached->is_size_known; + result.etag = cached->etag; + result.part_offsets = cached->part_offsets; + result.is_from_identity_cache = true; + if (cached->is_last_modified_known) + { + result.last_modified = Poco::Timestamp::fromEpochTime(static_cast(cached->last_modified)); + /// Real enough for `_time`, but not a freshness proof. The schema and count caches + /// treat an older time as "unchanged" and would keep a replaced object's schema and row count. + result.is_last_modified_fresh = false; + } + else + result.is_last_modified_known = false; + return result; } - else - result.is_last_modified_known = false; - return result; + + fill_last_modified = true; + preserved_etag = cached->etag; + preserved_part_offsets = std::move(cached->part_offsets); } } } @@ -1068,7 +1081,10 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool /// pay the heavier GetObjectAttributes, which also returns per-part sizes; it falls back to /// a HEAD internally if the API is unsupported or denied. This keeps a cold, cache-empty /// lake scan on the cheap-HEAD path instead of one GetObjectAttributes per file. - if (fetch_part_offsets) + /// An entry that already has part offsets but no Last-Modified is filled with a HEAD: this + /// SDK's GetObjectAttributes cannot request Last-Modified, and storing that as "unknown" + /// would make every later `_time` NULL. + if (fetch_part_offsets && !fill_last_modified) return S3::getObjectIdentity(c, uri.bucket, path, /*version_id=*/ {}); return S3::getObjectInfo(c, uri.bucket, path, /*version_id=*/ {}, /*with_metadata=*/ false, /*with_tags=*/ false); } @@ -1107,8 +1123,20 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool result.etag = object_info.etag; result.tags = std::move(object_info.tags); result.attributes = object_info.metadata; + /// Simulates a `GetObjectAttributes` success, which does not carry Last-Modified. The stored entry + /// must not report that as a known absence, or the next `_time` read stays NULL. + fiu_do_on(FailPoints::object_storage_identity_cache_skip_last_modified, { + object_info.last_modification_time = 0; + object_info.last_modification_time_observed = false; + result.last_modified = {}; + }); if (identity_only) - result.is_last_modified_known = object_info.last_modification_time != 0; + { + if (object_info.last_modification_time_observed) + result.is_last_modified_known = object_info.last_modification_time != 0; + else + result.is_last_modified_known = false; + } /// GetObjectAttributes returns per-part sizes; convert to cumulative start offsets. if (!object_info.part_sizes.empty()) @@ -1122,6 +1150,9 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool } } + if (result.part_offsets.empty() && !preserved_part_offsets.empty() && result.etag == preserved_etag) + result.part_offsets = std::move(preserved_part_offsets); + if (identity_cache) { /// The read that reached this point already holds the identity from before any overwrite that @@ -1132,8 +1163,9 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool identity.size = result.size_bytes; identity.is_size_known = result.is_size_known; identity.part_offsets = result.part_offsets; - identity.is_last_modified_known = result.is_last_modified_known; - if (result.is_last_modified_known) + identity.last_modified_observed = object_info.last_modification_time_observed; + identity.is_last_modified_known = identity.last_modified_observed && result.is_last_modified_known; + if (identity.is_last_modified_known) identity.last_modified = static_cast(result.last_modified.epochTime()); identity_cache->setIfEpoch(identity_key, std::move(identity), identity_read); } diff --git a/src/IO/S3/getObjectInfo.cpp b/src/IO/S3/getObjectInfo.cpp index 465cc4088da86..fcb216547f31e 100644 --- a/src/IO/S3/getObjectInfo.cpp +++ b/src/IO/S3/getObjectInfo.cpp @@ -128,6 +128,7 @@ namespace object_info.size = static_cast(result.GetContentLength()); object_info.is_size_known = result.ContentLengthHasBeenSet(); object_info.last_modification_time = result.GetLastModified().Seconds(); + object_info.last_modification_time_observed = true; object_info.etag = result.GetETag(); if (with_metadata) diff --git a/src/IO/S3/getObjectInfo.h b/src/IO/S3/getObjectInfo.h index 4eecfb0d101e9..5fa8c17d15294 100644 --- a/src/IO/S3/getObjectInfo.h +++ b/src/IO/S3/getObjectInfo.h @@ -18,6 +18,10 @@ struct ObjectInfo /// Checks if Content-Length was present in the HEAD response and we can rely on its result bool is_size_known = true; time_t last_modification_time = 0; + /// False when the response did not include Last-Modified. `GetObjectAttributes` in this SDK cannot + /// request it, so a successful attributes call leaves the time at 0 without having observed it. + /// A HEAD that returns epoch 0 sets this true: the object really has no modification time. + bool last_modification_time_observed = false; String etag; ObjectAttributes tags; // Set only if getObjectInfo() is called with `with_tags = true` ObjectAttributes metadata = {}; /// Set only if getObjectInfo() is called with `with_metadata = true`. diff --git a/tests/queries/0_stateless/05495_object_storage_identity_cache_part_offsets_time.reference b/tests/queries/0_stateless/05495_object_storage_identity_cache_part_offsets_time.reference new file mode 100644 index 0000000000000..d86bac9de59ab --- /dev/null +++ b/tests/queries/0_stateless/05495_object_storage_identity_cache_part_offsets_time.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/05495_object_storage_identity_cache_part_offsets_time.sh b/tests/queries/0_stateless/05495_object_storage_identity_cache_part_offsets_time.sh new file mode 100755 index 0000000000000..4e5c9603fa9a9 --- /dev/null +++ b/tests/queries/0_stateless/05495_object_storage_identity_cache_part_offsets_time.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-parallel-replicas +# Tag no-fasttest: reads a file through the mock S3 server at localhost:11111 +# Tag no-parallel: the identity cache and the failpoint are process-global +# Tag no-parallel-replicas: per-query S3 request counts change when a read is spread across replicas + +# GetObjectAttributes does not return Last-Modified. Caching that as "time unknown" makes every later +# `_time` NULL. The failpoint simulates that response. The next read, with part offsets off, must HEAD +# and return the real time. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +URL="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" +READ_SETTINGS="max_threads = 1, use_hive_partitioning = 0, parallel_replicas_for_cluster_engines = 0, enable_filesystem_cache = 0, use_page_cache_for_object_storage = 0, s3_validate_etag_on_read = 0, use_object_storage_list_objects_cache = 0" +FAILPOINT="object_storage_identity_cache_skip_last_modified" + +fail() +{ + echo "FAIL: $*" >&2 + exit 1 +} + +cleanup() +{ + ${CLICKHOUSE_CLIENT} -q "SYSTEM DISABLE FAILPOINT ${FAILPOINT}" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +read_time() +{ + local label=$1 + local extra_settings=$2 + local qid="idcache_time_${label}_${CLICKHOUSE_DATABASE}" + local value + + value=$(${CLICKHOUSE_CLIENT} --query_id "$qid" -q " + SELECT if(isNull(_time), 'NULL', toString(_time)) + FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') + SETTINGS ${READ_SETTINGS}${extra_settings} + FORMAT TabSeparated") || fail "${label}: select failed" + printf '%s' "$value" +} + +head_count() +{ + local qid=$1 + ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "flush logs failed" + ${CLICKHOUSE_CLIENT} -q " + SELECT ProfileEvents['S3HeadObject'] + FROM system.query_log + WHERE event_date >= yesterday() + AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND query_id = '${qid}' + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC + LIMIT 1 + FORMAT TabSeparated" +} + +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear failed" + +${CLICKHOUSE_CLIENT} -q " + INSERT INTO FUNCTION s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') + SELECT 1 + SETTINGS s3_truncate_on_insert = 1" || fail "insert failed" + +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear after insert failed" +${CLICKHOUSE_CLIENT} -q "SYSTEM ENABLE FAILPOINT ${FAILPOINT}" || fail "enable failpoint failed" + +without_time=$(read_time part_offsets ", object_storage_identity_cache_fetch_part_offsets = 1") +[[ $without_time == "NULL" ]] || fail "part-offset fetch stored a time '${without_time}', failpoint did not apply" + +${CLICKHOUSE_CLIENT} -q "SYSTEM DISABLE FAILPOINT ${FAILPOINT}" || fail "disable failpoint failed" + +later_qid="idcache_time_later_${CLICKHOUSE_DATABASE}" +later=$(${CLICKHOUSE_CLIENT} --query_id "$later_qid" -q " + SELECT if(isNull(_time), 'NULL', toString(_time)) + FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') + SETTINGS ${READ_SETTINGS} + FORMAT TabSeparated") || fail "later select failed" +[[ $later != "NULL" && $later != "1970-01-01 00:00:00" && -n $later ]] || fail "later _time is '${later}'" + +heads=$(head_count "$later_qid") +[[ $heads -ge 1 ]] || fail "later read did not HEAD (S3HeadObject=${heads})" + +echo OK From 24f0663153bf2d9148213d5a73c533703a1009bf Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 17 Nov 2026 19:55:27 +0100 Subject: [PATCH 22/37] Object storage: revalidate a filesystem-cache hit with HEAD, not GetObjectAttributes A filesystem or page cache hit re-reads the identity before using its ETag as the cache key. When the cached identity came with part offsets, that re-read used GetObjectAttributes. That call does not return Last-Modified, so the revalidating read reported `_time` as NULL and the store replaced the cached time with "not observed". The revalidation now uses a HEAD, which returns the ETag and Last-Modified. `ObjectStorageIdentityCache::setIfEpoch` keeps the part offsets already stored when the fetch saw the same ETag, because a HEAD does not return them. A different ETag still drops them. Adds test 05496 covering the primed, revalidated, and later reads. Co-authored-by: Cursor --- .../ObjectStorageIdentityCache.cpp | 11 +- .../ObjectStorageIdentityCache.h | 1 + .../StorageObjectStorageSource.cpp | 6 +- ...lesystem_cache_part_offsets_time.reference | 1 + ...ache_filesystem_cache_part_offsets_time.sh | 105 ++++++++++++++++++ 5 files changed, 120 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/05496_object_storage_identity_cache_filesystem_cache_part_offsets_time.reference create mode 100755 tests/queries/0_stateless/05496_object_storage_identity_cache_filesystem_cache_part_offsets_time.sh diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp index 3e6beb932fc2d..fbb22a805fe74 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp @@ -53,8 +53,15 @@ void ObjectStorageIdentityCache::setIfEpoch(const String & key, ObjectStorageIde if (epoch != read.epoch) return; /// `Base::get` takes the cache mutex after `epoch_mutex`, the same order as `set` and `remove`. - if (auto existing = Base::get(key); existing && existing->fetch_seq > read.seq) - return; + if (auto existing = Base::get(key)) + { + if (existing->fetch_seq > read.seq) + return; + /// A HEAD does not return multipart part offsets. Keep the ones already stored when this + /// fetch saw the same ETag. A different ETag is a different object, so its offsets are dropped. + if (identity.part_offsets.empty() && !existing->part_offsets.empty() && identity.etag == existing->etag) + identity.part_offsets = existing->part_offsets; + } identity.fetch_seq = read.seq; Base::set(key, std::make_shared(std::move(identity))); } diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h index b67f668848dc2..16670dd51cccf 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h @@ -104,6 +104,7 @@ class ObjectStorageIdentityCache Read beginFetch(); /// Store `identity` only when `read.epoch` is still current and no later fetch of `key` has stored. + /// A HEAD has no part offsets. When its ETag matches the resident entry, those offsets are kept. void setIfEpoch(const String & key, ObjectStorageIdentity identity, Read read); /// Advance the epoch and drop the entry. A fetch already in flight will not put the old identity back. diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index 8b452e150b2ce..0d8a797ae332c 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -1994,6 +1994,9 @@ std::unique_ptr createReadBuffer( /// A filesystem or page cache hit never issues the GET, so it cannot notice that the object was /// replaced. Re-read the identity before using its ETag as the cache key. An unchanged object keeps /// the same key and is served from the cache. A replaced object gets a new key and is read again. + /// This refresh is a HEAD. `GetObjectAttributes` does not return Last-Modified, so using it here + /// would make this read's `_time` NULL and replace the cached time with "not observed". Part offsets + /// already stored for the same ETag are kept by `setIfEpoch`. if ((use_filesystem_cache || use_page_cache) && object_info.metadata && object_info.metadata->is_from_identity_cache @@ -2001,8 +2004,7 @@ std::unique_ptr createReadBuffer( && object_info.metadata->isEtagUsableAsCacheKey()) { object_info.metadata = object_storage->getObjectMetadataIgnoringIdentityCache( - object_info, /*with_tags=*/ false, - /*fetch_part_offsets=*/ effective_read_settings.object_storage_identity_cache_fetch_part_offsets); + object_info, /*with_tags=*/ false, /*fetch_part_offsets=*/ false); } if (use_page_cache && !object_info.metadata->isEtagUsableAsCacheKey()) diff --git a/tests/queries/0_stateless/05496_object_storage_identity_cache_filesystem_cache_part_offsets_time.reference b/tests/queries/0_stateless/05496_object_storage_identity_cache_filesystem_cache_part_offsets_time.reference new file mode 100644 index 0000000000000..d86bac9de59ab --- /dev/null +++ b/tests/queries/0_stateless/05496_object_storage_identity_cache_filesystem_cache_part_offsets_time.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/05496_object_storage_identity_cache_filesystem_cache_part_offsets_time.sh b/tests/queries/0_stateless/05496_object_storage_identity_cache_filesystem_cache_part_offsets_time.sh new file mode 100755 index 0000000000000..01c76922baddf --- /dev/null +++ b/tests/queries/0_stateless/05496_object_storage_identity_cache_filesystem_cache_part_offsets_time.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-parallel-replicas +# Tag no-fasttest: reads a file through the mock S3 server at localhost:11111 +# Tag no-parallel: the identity cache and the filesystem cache are process-global +# Tag no-parallel-replicas: per-query S3 request counts change when a read is spread across replicas + +# A filesystem-cache revalidation must HEAD. GetObjectAttributes does not return Last-Modified, and +# storing that result makes this read's `_time` NULL and drops the time already in the identity cache. +# The mock S3 server rejects GetObjectAttributes and falls back to HEAD, so the assertion is the +# request itself: the revalidating read must not call GetObjectAttributes. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +URL="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" +COMMON_SETTINGS="max_threads = 1, use_hive_partitioning = 0, parallel_replicas_for_cluster_engines = 0, filesystem_cache_name = 'cache_for_readbigat', use_page_cache_for_object_storage = 0, s3_validate_etag_on_read = 1, use_object_storage_list_objects_cache = 0" + +fail() +{ + echo "FAIL: $*" >&2 + exit 1 +} + +# $1 label, $2 extra settings, $3 head mode, $4 attributes mode, $5 get mode, $6 cache mode +# Prints the _time value. +read_time() +{ + local label=$1 + local extra=$2 + local head_mode=$3 + local attributes_mode=$4 + local get_mode=$5 + local cache_mode=$6 + local qid="idcache_fscache_time_${label}_${CLICKHOUSE_DATABASE}" + local value row head attributes gets cache_bytes + + value=$(${CLICKHOUSE_CLIENT} --query_id "$qid" -q " + SELECT if(isNull(_time), 'NULL', toString(_time)) + FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') + SETTINGS ${COMMON_SETTINGS}${extra} + FORMAT TabSeparated") || fail "${label}: select failed" + + ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "${label}: flush logs failed" + row=$(${CLICKHOUSE_CLIENT} -q " + SELECT + ProfileEvents['S3HeadObject'], + ProfileEvents['S3GetObjectAttributes'], + ProfileEvents['S3GetObject'], + ProfileEvents['CachedReadBufferReadFromCacheBytes'] + FROM system.query_log + WHERE event_date >= yesterday() + AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND query_id = '${qid}' + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC + LIMIT 1 + FORMAT TabSeparated") || fail "${label}: query_log failed" + [[ -n $row ]] || fail "${label}: no query_log row" + + IFS=$'\t' read -r head attributes gets cache_bytes <<< "$row" + case "$head_mode" in + eq0) [[ $head == "0" ]] || fail "${label}: S3HeadObject=${head}, expected 0" ;; + ge1) [[ $head -ge 1 ]] || fail "${label}: S3HeadObject=${head}, expected >= 1" ;; + *) fail "${label}: bad head mode ${head_mode}" ;; + esac + case "$attributes_mode" in + eq0) [[ $attributes == "0" ]] || fail "${label}: S3GetObjectAttributes=${attributes}, expected 0" ;; + *) fail "${label}: bad attributes mode ${attributes_mode}" ;; + esac + case "$get_mode" in + eq0) [[ $gets == "0" ]] || fail "${label}: S3GetObject=${gets}, expected 0" ;; + ge1) [[ $gets -ge 1 ]] || fail "${label}: S3GetObject=${gets}, expected >= 1" ;; + *) fail "${label}: bad get mode ${get_mode}" ;; + esac + case "$cache_mode" in + ge1) [[ $cache_bytes -ge 1 ]] || fail "${label}: CachedReadBufferReadFromCacheBytes=${cache_bytes}, expected >= 1" ;; + any) ;; + *) fail "${label}: bad cache mode ${cache_mode}" ;; + esac + + printf '%s' "$value" +} + +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear identity cache failed" +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR FILESYSTEM CACHE 'cache_for_readbigat'" || fail "clear filesystem cache failed" + +${CLICKHOUSE_CLIENT} -q " + INSERT INTO FUNCTION s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') + SELECT 1 + SETTINGS s3_truncate_on_insert = 1" || fail "insert failed" + +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear identity cache after insert failed" + +primed=$(read_time primed ", enable_filesystem_cache = 1, object_storage_identity_cache_fetch_part_offsets = 0" ge1 eq0 ge1 any) +[[ $primed != "NULL" && $primed != "1970-01-01 00:00:00" && -n $primed ]] || fail "primed _time is '${primed}'" + +revalidated=$(read_time revalidated ", enable_filesystem_cache = 1, object_storage_identity_cache_fetch_part_offsets = 1" ge1 eq0 eq0 ge1) +[[ $revalidated == "$primed" ]] || fail "revalidated _time is '${revalidated}', primed is '${primed}'" + +later=$(read_time later ", enable_filesystem_cache = 0, object_storage_identity_cache_fetch_part_offsets = 0" eq0 eq0 ge1 any) +[[ $later == "$primed" ]] || fail "later _time is '${later}', primed is '${primed}'" + +echo OK From 5d910de63956d55f53131315553de4e201eb95c6 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 17 Nov 2026 20:06:08 +0100 Subject: [PATCH 23/37] Object storage: HEAD before a Parquet footer-only count uses an identity-cache ETag A footer-only Parquet `count()` reads the row count from the parquet metadata cache and never issues a GET. `ParquetMetadataCache` keys by ETag, and the ETag came from the identity cache. After an external replace, the count returned the old row count, and `s3_validate_etag_on_read` never ran because no GET was sent. When the read is a Parquet count and `use_parquet_metadata_cache` is enabled, `createReadBuffer` now revalidates an identity-cache ETag with a HEAD, using the same path as the filesystem and page caches. Parquet reads that GET the data still check the ETag there and are unchanged. Adds test 05497 that counts a Parquet object, replaces it externally, and expects the next count to return the new row count after a HEAD. Co-authored-by: Cursor --- .../StorageObjectStorageSource.cpp | 27 ++++++-- src/Storages/ObjectStorage/Utils.h | 3 +- ...ache_parquet_count_after_replace.reference | 1 + ...ntity_cache_parquet_count_after_replace.sh | 61 +++++++++++++++++++ 4 files changed, 85 insertions(+), 7 deletions(-) create mode 100644 tests/queries/0_stateless/05497_object_storage_identity_cache_parquet_count_after_replace.reference create mode 100755 tests/queries/0_stateless/05497_object_storage_identity_cache_parquet_count_after_replace.sh diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index 0d8a797ae332c..7a5b85fa5a3a3 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -1462,8 +1462,21 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade { ProfileEvents::increment(ProfileEvents::ObjectStorageReadObjects); compression_method = chooseCompressionMethod(object_info->getFileName(), configuration->compression_method); + /// A Parquet `count()` reads the cached footer and does not GET, so `If-Match` never + /// rejects a replaced object. Re-HEAD before that ETag keys the parquet metadata cache. + /// A read that does GET still checks the ETag there and does not pay this HEAD. + const bool revalidate_identity_for_parquet_metadata_cache = + effective_need_only_count + && context_->getSettingsRef()[Setting::use_parquet_metadata_cache] + && Poco::toLower(format_name) == "parquet"; read_buf = createReadBuffer( - object_info->relative_path_with_metadata, object_info->getResolvedStorage(object_storage), context_, log, std::nullopt, !headers_requested); + object_info->relative_path_with_metadata, + object_info->getResolvedStorage(object_storage), + context_, + log, + std::nullopt, + !headers_requested, + revalidate_identity_for_parquet_metadata_cache); } Block initial_header = read_from_format_info.format_header; @@ -1933,7 +1946,8 @@ std::unique_ptr createReadBuffer( const ContextPtr & context_, const LoggerPtr & log, const std::optional & read_settings, - bool allow_page_cache) + bool allow_page_cache, + bool revalidate_identity_for_parquet_metadata_cache) { const auto & settings = context_->getSettingsRef(); const auto & effective_read_settings = read_settings.has_value() ? read_settings.value() : context_->getReadSettings(); @@ -1991,13 +2005,14 @@ std::unique_ptr createReadBuffer( object_info.metadata = object_storage->getObjectMetadata(object_info, /*with_tags=*/ false); } - /// A filesystem or page cache hit never issues the GET, so it cannot notice that the object was - /// replaced. Re-read the identity before using its ETag as the cache key. An unchanged object keeps - /// the same key and is served from the cache. A replaced object gets a new key and is read again. + /// A filesystem, page, or parquet-metadata cache hit never issues the GET, so it cannot notice + /// that the object was replaced. Re-read the identity before using its ETag as the cache key. + /// An unchanged object keeps the same key. A replaced object gets a new key and is read again. + /// The parquet flag is only a footer-only count: other Parquet reads still check the ETag on GET. /// This refresh is a HEAD. `GetObjectAttributes` does not return Last-Modified, so using it here /// would make this read's `_time` NULL and replace the cached time with "not observed". Part offsets /// already stored for the same ETag are kept by `setIfEpoch`. - if ((use_filesystem_cache || use_page_cache) + if ((use_filesystem_cache || use_page_cache || revalidate_identity_for_parquet_metadata_cache) && object_info.metadata && object_info.metadata->is_from_identity_cache && object_info.metadata->is_size_known diff --git a/src/Storages/ObjectStorage/Utils.h b/src/Storages/ObjectStorage/Utils.h index 096b401f9852e..49a8f8b05b5be 100644 --- a/src/Storages/ObjectStorage/Utils.h +++ b/src/Storages/ObjectStorage/Utils.h @@ -37,7 +37,8 @@ std::unique_ptr createReadBuffer( const ContextPtr & context_, const LoggerPtr & log, const std::optional & read_settings = std::nullopt, - bool allow_page_cache = true); + bool allow_page_cache = true, + bool revalidate_identity_for_parquet_metadata_cache = false); ASTs::iterator getFirstKeyValueArgument(ASTs & args); std::unordered_map parseKeyValueArguments(const ASTs & function_args, ContextPtr context); diff --git a/tests/queries/0_stateless/05497_object_storage_identity_cache_parquet_count_after_replace.reference b/tests/queries/0_stateless/05497_object_storage_identity_cache_parquet_count_after_replace.reference new file mode 100644 index 0000000000000..d86bac9de59ab --- /dev/null +++ b/tests/queries/0_stateless/05497_object_storage_identity_cache_parquet_count_after_replace.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/05497_object_storage_identity_cache_parquet_count_after_replace.sh b/tests/queries/0_stateless/05497_object_storage_identity_cache_parquet_count_after_replace.sh new file mode 100755 index 0000000000000..3c9425ca2ab64 --- /dev/null +++ b/tests/queries/0_stateless/05497_object_storage_identity_cache_parquet_count_after_replace.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-parallel-replicas +# Tag no-fasttest: reads a file through the mock S3 server at localhost:11111 +# Tag no-parallel: the identity cache and the parquet metadata cache are process-global +# Tag no-parallel-replicas: per-query S3 request counts change when a read is spread across replicas + +# A Parquet count reads the footer from the parquet metadata cache and does not GET. The cache key +# includes the object ETag. An identity-cache hit must not supply that ETag until a HEAD confirms it, +# or the count keeps the replaced object's row count. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +URL="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}.parquet" +READ_SETTINGS="max_threads = 1, use_hive_partitioning = 0, parallel_replicas_for_cluster_engines = 0, enable_filesystem_cache = 0, use_page_cache_for_object_storage = 0, s3_validate_etag_on_read = 1, use_object_storage_list_objects_cache = 0, use_cache_for_count_from_files = 1, schema_inference_use_cache_for_s3 = 1, use_parquet_metadata_cache = 1" + +fail() +{ + echo "FAIL: $*" >&2 + exit 1 +} + +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear identity cache failed" +${CLICKHOUSE_CLIENT} -q "SYSTEM DROP PARQUET METADATA CACHE" || fail "drop parquet metadata cache failed" + +${CLICKHOUSE_CLIENT} -q " + INSERT INTO FUNCTION s3('${URL}', 'test', 'testtest', 'Parquet', 'n UInt64') + SELECT toUInt64(1) + SETTINGS s3_truncate_on_insert = 1" || fail "insert failed" + +count1=$(${CLICKHOUSE_CLIENT} -q " + SELECT count() FROM s3('${URL}', 'test', 'testtest', 'Parquet', 'n UInt64') + SETTINGS ${READ_SETTINGS}") || fail "first count failed" +[[ $count1 == "1" ]] || fail "first count '${count1}', expected 1" + +payload="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}.parquet" +${CLICKHOUSE_CLIENT} -q "SELECT toUInt64(number) AS n FROM numbers(3) FORMAT Parquet" > "$payload" || fail "local parquet failed" +curl -fsS -X PUT "$URL" --upload-file "$payload" >/dev/null || fail "external put failed" + +replaced_qid="idcache_parquet_count_${CLICKHOUSE_DATABASE}" +count2=$(${CLICKHOUSE_CLIENT} --query_id "$replaced_qid" -q " + SELECT count() FROM s3('${URL}', 'test', 'testtest', 'Parquet', 'n UInt64') + SETTINGS ${READ_SETTINGS}") || fail "replaced count failed" +[[ $count2 == "3" ]] || fail "replaced count '${count2}', expected 3" + +${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "flush logs failed" +heads=$(${CLICKHOUSE_CLIENT} -q " + SELECT ProfileEvents['S3HeadObject'] + FROM system.query_log + WHERE event_date >= yesterday() + AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND query_id = '${replaced_qid}' + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC + LIMIT 1 + FORMAT TabSeparated") || fail "query_log failed" +[[ $heads -ge 1 ]] || fail "replaced count did not HEAD (S3HeadObject=${heads})" + +echo OK From 62901c490fd6a7a6cefaf29f0c6d1e572b84d29c Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 17 Nov 2026 20:20:21 +0100 Subject: [PATCH 24/37] Object storage: identity cache stores only HEAD results The identity cache had a second source of identity: `GetObjectAttributes`, which carried multipart part offsets, and a `last_modified_observed` flag because that request does not return Last-Modified. That path was not in any release, so it is removed rather than kept as dead complexity. Every identity-cache entry now comes from a HEAD, which always returns size, ETag and Last-Modified. `getObjectMetadata` no longer takes `fetch_part_offsets`, `setIfEpoch` only checks the epoch and `fetch_seq`, and the `object_storage_identity_cache_fetch_part_offsets` setting, the `object_storage_identity_cache_skip_last_modified` failpoint, and the `S3GetObjectAttributes` and `DiskS3GetObjectAttributes` events are removed. The S3 client's `getObjectIdentity` wrapper is removed. The fixes from earlier commits are kept: a cache hit reports Last-Modified with `is_last_modified_fresh` set to false, and filesystem, page and Parquet footer-only count revalidation issue a HEAD before using a cached ETag. Tests 05495 and 05496 covered only the part-offset path and are removed. Tests 05484, 05485, 05486 and 05490 no longer count `GetObjectAttributes`. Co-authored-by: Cursor --- docs/reference/statements/system.mdx | 2 +- src/Common/FailPoint.cpp | 1 - src/Common/ProfileEvents.cpp | 4 +- src/Core/ServerSettings.cpp | 2 +- src/Core/Settings.cpp | 3 - src/Core/SettingsChangesHistory.cpp | 1 - .../ObjectStorages/IObjectStorage.h | 27 +---- .../ObjectStorageIdentityCache.cpp | 11 +- .../ObjectStorageIdentityCache.h | 20 +--- .../ObjectStorages/S3/S3ObjectStorage.cpp | 104 +++++------------ .../ObjectStorages/S3/S3ObjectStorage.h | 20 +--- src/IO/ReadSettings.h | 4 - src/IO/S3/Client.cpp | 6 - src/IO/S3/Client.h | 2 - src/IO/S3/Requests.h | 2 - src/IO/S3/getObjectInfo.cpp | 71 ----------- src/IO/S3/getObjectInfo.h | 21 ---- src/Interpreters/Context.cpp | 4 +- src/Interpreters/Context.h | 2 +- .../Iceberg/IcebergDataObjectInfo.cpp | 2 +- .../StorageObjectStorageSource.cpp | 29 ++--- .../StorageObjectStorageSource.h | 4 +- .../05484_object_storage_identity_cache.sh | 34 ++---- ...ect_storage_identity_cache_endpoint_key.sh | 6 +- ...object_storage_identity_cache_overwrite.sh | 6 +- ...age_identity_cache_etag_spelling.reference | 1 - ...ct_storage_identity_cache_etag_spelling.sh | 110 ------------------ ...storage_identity_cache_external_replace.sh | 6 +- ...identity_cache_part_offsets_time.reference | 1 - ...torage_identity_cache_part_offsets_time.sh | 89 -------------- ...lesystem_cache_part_offsets_time.reference | 1 - ...ache_filesystem_cache_part_offsets_time.sh | 105 ----------------- 32 files changed, 73 insertions(+), 628 deletions(-) delete mode 100644 tests/queries/0_stateless/05487_object_storage_identity_cache_etag_spelling.reference delete mode 100755 tests/queries/0_stateless/05487_object_storage_identity_cache_etag_spelling.sh delete mode 100644 tests/queries/0_stateless/05495_object_storage_identity_cache_part_offsets_time.reference delete mode 100755 tests/queries/0_stateless/05495_object_storage_identity_cache_part_offsets_time.sh delete mode 100644 tests/queries/0_stateless/05496_object_storage_identity_cache_filesystem_cache_part_offsets_time.reference delete mode 100755 tests/queries/0_stateless/05496_object_storage_identity_cache_filesystem_cache_part_offsets_time.sh diff --git a/docs/reference/statements/system.mdx b/docs/reference/statements/system.mdx index fc8e43c893b21..4485881efcb0a 100644 --- a/docs/reference/statements/system.mdx +++ b/docs/reference/statements/system.mdx @@ -136,7 +136,7 @@ Clears the parquet metadata cache. ## SYSTEM CLEAR|DROP OBJECT STORAGE IDENTITY CACHE {#drop-object-storage-identity-cache} -Clears the object-storage identity cache, which stores per-object size, ETag and multipart part offsets so that opening an object does not repeat a `HEAD` or `GetObjectAttributes` request. Its size is configured with the server settings `object_storage_identity_cache_size`, `object_storage_identity_cache_policy` and `object_storage_identity_cache_size_ratio`. Set `object_storage_identity_cache_size` to `0` to disable the cache. `SYSTEM RELOAD CONFIG` with a non-zero size creates it when startup left it disabled. A write, copy, or delete by this server drops that object's entry, and the next read requests the metadata again. +Clears the object-storage identity cache, which stores per-object size, ETag and Last-Modified so that opening an object does not repeat a `HEAD` request. Its size is configured with the server settings `object_storage_identity_cache_size`, `object_storage_identity_cache_policy` and `object_storage_identity_cache_size_ratio`. Set `object_storage_identity_cache_size` to `0` to disable the cache. `SYSTEM RELOAD CONFIG` with a non-zero size creates it when startup left it disabled. A write, copy, or delete by this server drops that object's entry, and the next read requests the metadata again. ## SYSTEM CLEAR|DROP PAIMON METADATA CACHE {#drop-paimon-metadata-cache} diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp index 1d13be3451a9f..c7c49cfe8c718 100644 --- a/src/Common/FailPoint.cpp +++ b/src/Common/FailPoint.cpp @@ -85,7 +85,6 @@ static struct InitFiu ONCE(s3_send_request_throw_expired_token) \ REGULAR(s3_read_inject_etag_mismatch) \ PAUSEABLE_ONCE(object_storage_identity_cache_pause_before_store) \ - REGULAR(object_storage_identity_cache_skip_last_modified) \ REGULAR(azure_inject_forbidden_response) \ ONCE(azure_inject_forbidden_response_once) \ REGULAR(azure_inject_auth_failure_on_request) \ diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 5753a27618bb6..7ed0c0fe77f9d 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -820,8 +820,7 @@ The server successfully detected this situation and will download merged part fr M(S3ListObjects, "Number of S3 API ListObjects calls.", ValueType::Number) \ M(S3ListObjectsMicroseconds, "Time of S3 API ListObjects execution.", ValueType::Microseconds) \ M(S3HeadObject, "Number of S3 API HeadObject calls.", ValueType::Number) \ - M(S3GetObjectAttributes, "Number of S3 API GetObjectAttributes calls.", ValueType::Number) \ - M(ObjectStorageIdentityCacheHits, "Number of object-storage identity (size/etag/part-offsets) cache hits, avoiding a HEAD/GetObjectAttributes.", ValueType::Number) \ + M(ObjectStorageIdentityCacheHits, "Number of object-storage identity (size/etag/Last-Modified) cache hits, avoiding a HEAD.", ValueType::Number) \ M(ObjectStorageIdentityCacheMisses, "Number of object-storage identity cache misses that triggered a metadata request.", ValueType::Number) \ /* CA per-namespace S3 op instrumentation: one event per (namespace, op) cell of the table in */ \ /* CasInstrumentedBackend.cpp; attributes PUTs/HEADs/404s/412s/LISTs to the CA operation type. */ \ @@ -1025,7 +1024,6 @@ The server successfully detected this situation and will download merged part fr M(DiskS3CopyObject, "Number of DiskS3 API CopyObject calls.", ValueType::Number) \ M(DiskS3ListObjects, "Number of DiskS3 API ListObjects calls.", ValueType::Number) \ M(DiskS3HeadObject, "Number of DiskS3 API HeadObject calls.", ValueType::Number) \ - M(DiskS3GetObjectAttributes, "Number of DiskS3 API GetObjectAttributes calls.", ValueType::Number) \ M(DiskS3GetObjectTagging, "Number of DiskS3 API GetObjectTagging calls.", ValueType::Number) \ M(DiskS3CreateMultipartUpload, "Number of DiskS3 API CreateMultipartUpload calls.", ValueType::Number) \ M(DiskS3UploadPartCopy, "Number of DiskS3 API UploadPartCopy calls.", ValueType::Number) \ diff --git a/src/Core/ServerSettings.cpp b/src/Core/ServerSettings.cpp index aa18e5b4c8117..e259ae6fe46fb 100644 --- a/src/Core/ServerSettings.cpp +++ b/src/Core/ServerSettings.cpp @@ -633,7 +633,7 @@ This setting can be modified at runtime and will take effect immediately. DECLARE(Double, mark_cache_size_ratio, DEFAULT_MARK_CACHE_SIZE_RATIO, R"(The size of the protected queue (in case of SLRU policy) in the mark cache relative to the cache's total size.)", 0) \ DECLARE(Double, mark_cache_prewarm_ratio, 0.95, R"(The ratio of total size of mark cache to fill during prewarm.)", 0) \ DECLARE(String, object_storage_identity_cache_policy, "SLRU", R"(Object-storage identity cache policy name (SLRU or LRU).)", 0) \ - DECLARE(UInt64, object_storage_identity_cache_size, 67108864, R"(Maximum size of the object-storage identity cache in bytes. This cache stores per-object size, ETag and multipart part offsets so that opening an object does not require a fresh HEAD/GetObjectAttributes on every read. Set to 0 to disable. A reload (`SYSTEM RELOAD CONFIG`) with a non-zero size creates the cache when startup left it disabled, so a restart is not required to turn it on.)", 0) \ + DECLARE(UInt64, object_storage_identity_cache_size, 67108864, R"(Maximum size of the object-storage identity cache in bytes. This cache stores per-object size, ETag and Last-Modified so that opening an object does not require a fresh HEAD on every read. Set to 0 to disable. A reload (`SYSTEM RELOAD CONFIG`) with a non-zero size creates the cache when startup left it disabled, so a restart is not required to turn it on.)", 0) \ DECLARE(Double, object_storage_identity_cache_size_ratio, 0.5, R"(The size of the protected queue (in case of SLRU policy) in the object-storage identity cache relative to the cache's total size.)", 0) \ DECLARE(String, unique_key_index_cache_policy, "SLRU", R"(UNIQUE KEY index cache policy name (SLRU or LRU).)", 0) \ DECLARE(UInt64, unique_key_index_cache_size_bytes, 1_GiB, R"(Maximum size (bytes) of the in-process cache for UNIQUE KEY index (SST) blocks. Set to 0 to disable the cache.)", 0) \ diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 16e97fc4b48e5..4150173e8bb28 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -7347,9 +7347,6 @@ Use userspace page cache when reading from local disks. Used for testing, unlike )", 0) \ DECLARE(Bool, use_page_cache_for_object_storage, false, R"( Use userspace page cache when reading from object storage table functions (s3, azure, hdfs) and table engines (S3, Azure, HDFS). -)", 0) \ - DECLARE(Bool, object_storage_identity_cache_fetch_part_offsets, false, R"( -Fetch the S3 multipart part layout (part offsets) when reading object metadata, using a `GetObjectAttributes` request instead of a plain `HEAD`. The part offsets enable part-aligned reads, but `GetObjectAttributes` is heavier than a `HEAD` and is issued per file, so on a cold, cache-empty scan it adds latency without benefit unless part-aligned reads are used. When disabled (the default), a `HEAD` is used, which still returns the object size and etag needed for cache keys. )", 0) \ DECLARE(Bool, read_from_page_cache_if_exists_otherwise_bypass_cache, false, R"( Use userspace page cache in passive mode, similar to read_from_filesystem_cache_if_exists_otherwise_bypass_cache. diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index ac666884dfc7f..1864edcad4b3e 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -52,7 +52,6 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() }); addSettingsChanges(settings_changes_history, "26.8.34.20001.altinityantalya", { - {"object_storage_identity_cache_fetch_part_offsets", false, false, "New setting: fetch the S3 multipart part layout via GetObjectAttributes when reading object metadata (for part-aligned reads). Disabled by default, in which case a cheaper HEAD (size + etag) is used, avoiding a per-file GetObjectAttributes on cold lake scans."}, {"input_format_parquet_detect_variant_by_structure", false, true, "New setting to read Spark variant columns, which have no `VARIANT` logical type, as `Dynamic` instead of `Tuple`"}, {"use_iceberg_manifest_list_partition_pruning", false, true, "New setting to skip Iceberg manifest files whose manifest-list partition summaries cannot match the query filter, without reading them."}, }); diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h b/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h index ccd6cdc1fa943..162cd6852da4e 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h @@ -163,10 +163,6 @@ struct ObjectMetadata bool etag_is_strong = true; ObjectAttributes tags; ObjectAttributes attributes; - /// Cumulative start offsets of the object's multipart-upload parts (part i covers - /// [part_offsets[i], part_offsets[i+1])). Populated best-effort via GetObjectAttributes; empty - /// when unknown or single-PUT. Used to align reads to part boundaries. - std::vector part_offsets = {}; /// `etag` may be used as a content-cache key (filesystem cache, page cache, Parquet /// metadata cache) only when it is present and a strong content identifier. A weak @@ -376,31 +372,12 @@ class IObjectStorage return getObjectMetadata(object.getPath(), with_tags); } - /// Same as above, but lets the caller ask for the object's multipart part layout (part_offsets). - /// Only S3 populates it, and only on request: it costs a heavier GetObjectAttributes call instead - /// of a plain HEAD (both return size + etag; only GetObjectAttributes returns the per-part sizes). - /// The default ignores the flag and issues the plain metadata request, so non-S3 storages and - /// callers that do not need part-aligned reads pay nothing extra. - virtual ObjectMetadata getObjectMetadata(const std::string & path, bool with_tags, bool /*fetch_part_offsets*/) const - { - return getObjectMetadata(path, with_tags); - } - - /// Same request as the string overload, but the object keeps whatever identifies the replica - /// (a `url()` shard index). The default calls the two-argument object overload so a backend that - /// overrides that one, such as `WebObjectStorage`, still sees the index. Calling the string - /// overload instead would probe every shard and could return another shard's size and ETag. - virtual ObjectMetadata getObjectMetadata(const RelativePathWithMetadata & object, bool with_tags, bool /*fetch_part_offsets*/) const - { - return getObjectMetadata(object, with_tags); - } - /// Same as `getObjectMetadata`, but do not answer from the identity cache. A content cache (filesystem /// cache, page cache) keyed by a cached ETag would otherwise serve a replaced object's old bytes, because /// a full hit never issues the GET that compares ETags. The default has no identity cache. - virtual ObjectMetadata getObjectMetadataIgnoringIdentityCache(const RelativePathWithMetadata & object, bool with_tags, bool fetch_part_offsets) const + virtual ObjectMetadata getObjectMetadataIgnoringIdentityCache(const RelativePathWithMetadata & object, bool with_tags) const { - return getObjectMetadata(object, with_tags, fetch_part_offsets); + return getObjectMetadata(object, with_tags); } /// Same as getObjectMetadata(), but ignores if object does not exist. diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp index fbb22a805fe74..3e6beb932fc2d 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp @@ -53,15 +53,8 @@ void ObjectStorageIdentityCache::setIfEpoch(const String & key, ObjectStorageIde if (epoch != read.epoch) return; /// `Base::get` takes the cache mutex after `epoch_mutex`, the same order as `set` and `remove`. - if (auto existing = Base::get(key)) - { - if (existing->fetch_seq > read.seq) - return; - /// A HEAD does not return multipart part offsets. Keep the ones already stored when this - /// fetch saw the same ETag. A different ETag is a different object, so its offsets are dropped. - if (identity.part_offsets.empty() && !existing->part_offsets.empty() && identity.etag == existing->etag) - identity.part_offsets = existing->part_offsets; - } + if (auto existing = Base::get(key); existing && existing->fetch_seq > read.seq) + return; identity.fetch_seq = read.seq; Base::set(key, std::make_shared(std::move(identity))); } diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h index 16670dd51cccf..bed0ed4c20642 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h @@ -6,40 +6,33 @@ #include #include #include -#include namespace DB { /// Object-storage "identity": the bits needed to open an object without a fresh HEAD - its size, -/// ETag, Last-Modified, and (if multipart-uploaded) the byte offsets of its parts. +/// ETag and Last-Modified. /// /// It exists to break the ordering that forces a HEAD before every open: object caches key on /// (path, etag), so the etag must be known up front. Here the key is the store plus the path -/// (`makeObjectStorageIdentityCacheKey`) and the etag is stored in the value, learned once (via -/// GetObjectAttributes / a GET response) and reused. Multipart part offsets ride along so reads can -/// be aligned to part boundaries without re-probing. +/// (`makeObjectStorageIdentityCacheKey`) and the etag is stored in the value, learned once (via a +/// HEAD or a GET response) and reused. Every entry comes from a HEAD, which always returns +/// Last-Modified, so `is_last_modified_known` only means "the object has a modification time". struct ObjectStorageIdentity { String etag; UInt64 size = 0; bool is_size_known = true; - /// Cumulative start offset of each multipart part (part i covers [part_offsets[i], part_offsets[i+1])). - /// Empty when unknown or the object is a single PUT. - std::vector part_offsets; /// Seconds since epoch. Meaningful only when `is_last_modified_known`. Int64 last_modified = 0; bool is_last_modified_known = false; - /// False when this entry was stored from `GetObjectAttributes`, which does not return Last-Modified. - /// That is not the same as observing that the object has no modification time. - bool last_modified_observed = false; /// Orders stores of this key. A fetch that started earlier must not overwrite one that started later. /// Not part of the object's identity. UInt64 fetch_seq = 0; size_t memoryUsage() const { - return sizeof(ObjectStorageIdentity) + etag.capacity() + part_offsets.capacity() * sizeof(UInt64); + return sizeof(ObjectStorageIdentity) + etag.capacity(); } }; @@ -70,7 +63,7 @@ inline String makeObjectStorageIdentityCacheKey(std::string_view endpoint, std:: } /// Cache of object-storage identity, keyed by endpoint, bucket and object path, so opening an object -/// does not require a fresh HEAD / GetObjectAttributes on every read (a lake scan otherwise re-HEADs +/// does not require a fresh HEAD on every read (a lake scan otherwise re-HEADs /// every candidate file on every query). Backed by CacheBase (LRU/SLRU) for size-bounded eviction /// and thread-safety. /// @@ -104,7 +97,6 @@ class ObjectStorageIdentityCache Read beginFetch(); /// Store `identity` only when `read.epoch` is still current and no later fetch of `key` has stored. - /// A HEAD has no part offsets. When its ETag matches the resident entry, those offsets are kept. void setIfEpoch(const String & key, ObjectStorageIdentity identity, Read read); /// Advance the epoch and drop the entry. A fetch already in flight will not put the old identity back. diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp index ef3b612567a21..6ae9b12c54e19 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp @@ -64,7 +64,6 @@ namespace DB::FailPoints { extern const char object_storage_force_refresh_callback_success[]; extern const char object_storage_identity_cache_pause_before_store[]; - extern const char object_storage_identity_cache_skip_last_modified[]; } @@ -1004,13 +1003,16 @@ std::optional S3ObjectStorage::tryGetObjectMetadataImpl( return result; } -ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool with_tags, bool fetch_part_offsets, bool use_identity_cache) const +ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool with_tags) const { - /// When the caller only needs identity (size / etag / part offsets, not tags or user metadata) we - /// take a fast path: consult the identity cache to skip the metadata request entirely on repeat - /// opens. A miss uses a plain HEAD (size + etag) unless `fetch_part_offsets` is set, in which case - /// GetObjectAttributes also returns multipart part sizes and falls back to HEAD. This is what - /// removes the per-file metadata request on lake scans. The tags path is unchanged. + return getObjectMetadata(path, with_tags, /*use_identity_cache=*/ true); +} + +ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool with_tags, bool use_identity_cache) const +{ + /// When the caller only needs identity (size / etag / Last-Modified, not tags or user metadata) we + /// take a fast path: consult the identity cache to skip the HEAD on repeat opens. A miss issues a + /// plain HEAD. This is what removes the per-file metadata request on lake scans. The tags path is unchanged. /// See ObjectStorageIdentityCache. /// /// The cache is owned by the global Context and sized from server settings. Low-level @@ -1019,12 +1021,6 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool const bool identity_only = !with_tags; /// Bucket and path alone collide when two endpoints reuse a bucket name. const String identity_key = makeObjectStorageIdentityCacheKey(uri.endpoint, uri.bucket, path); - /// A `GetObjectAttributes` entry has size, ETag and part offsets, but this SDK cannot ask it for - /// Last-Modified. Keep those offsets and HEAD for the time instead of caching "time unknown". - bool fill_last_modified = false; - String preserved_etag; - std::vector preserved_part_offsets; - ObjectStorageIdentityCachePtr identity_cache; if (identity_only) { @@ -1035,29 +1031,21 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool { if (auto cached = identity_cache->tryGet(identity_key)) { - if (cached->last_modified_observed) + ObjectMetadata result; + result.size_bytes = cached->size; + result.is_size_known = cached->is_size_known; + result.etag = cached->etag; + result.is_from_identity_cache = true; + if (cached->is_last_modified_known) { - ObjectMetadata result; - result.size_bytes = cached->size; - result.is_size_known = cached->is_size_known; - result.etag = cached->etag; - result.part_offsets = cached->part_offsets; - result.is_from_identity_cache = true; - if (cached->is_last_modified_known) - { - result.last_modified = Poco::Timestamp::fromEpochTime(static_cast(cached->last_modified)); - /// Real enough for `_time`, but not a freshness proof. The schema and count caches - /// treat an older time as "unchanged" and would keep a replaced object's schema and row count. - result.is_last_modified_fresh = false; - } - else - result.is_last_modified_known = false; - return result; + result.last_modified = Poco::Timestamp::fromEpochTime(static_cast(cached->last_modified)); + /// Real enough for `_time`, but not a freshness proof. The schema and count caches + /// treat an older time as "unchanged" and would keep a replaced object's schema and row count. + result.is_last_modified_fresh = false; } - - fill_last_modified = true; - preserved_etag = cached->etag; - preserved_part_offsets = std::move(cached->part_offsets); + else + result.is_last_modified_known = false; + return result; } } } @@ -1073,21 +1061,9 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool auto fetch = [&](const S3::Client & c) -> S3::ObjectInfo { + /// A plain HEAD returns size, ETag and Last-Modified, which is everything the identity cache stores. if (identity_only) - { - /// Default (fetch_part_offsets = false): a plain HEAD returns size + etag, which is all the - /// read path needs (etag keys the page / filesystem / parquet-metadata caches, size drives - /// prefetch). Only when the caller wants the multipart layout for part-aligned reads do we - /// pay the heavier GetObjectAttributes, which also returns per-part sizes; it falls back to - /// a HEAD internally if the API is unsupported or denied. This keeps a cold, cache-empty - /// lake scan on the cheap-HEAD path instead of one GetObjectAttributes per file. - /// An entry that already has part offsets but no Last-Modified is filled with a HEAD: this - /// SDK's GetObjectAttributes cannot request Last-Modified, and storing that as "unknown" - /// would make every later `_time` NULL. - if (fetch_part_offsets && !fill_last_modified) - return S3::getObjectIdentity(c, uri.bucket, path, /*version_id=*/ {}); return S3::getObjectInfo(c, uri.bucket, path, /*version_id=*/ {}, /*with_metadata=*/ false, /*with_tags=*/ false); - } return S3::getObjectInfo(c, uri.bucket, path, /*version_id=*/ {}, /*with_metadata=*/ true, /*with_tags=*/ with_tags); }; @@ -1123,35 +1099,9 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool result.etag = object_info.etag; result.tags = std::move(object_info.tags); result.attributes = object_info.metadata; - /// Simulates a `GetObjectAttributes` success, which does not carry Last-Modified. The stored entry - /// must not report that as a known absence, or the next `_time` read stays NULL. - fiu_do_on(FailPoints::object_storage_identity_cache_skip_last_modified, { - object_info.last_modification_time = 0; - object_info.last_modification_time_observed = false; - result.last_modified = {}; - }); + /// Epoch 0 means the response had no Last-Modified. Keep it out of `_time` and the schema and count caches. if (identity_only) - { - if (object_info.last_modification_time_observed) - result.is_last_modified_known = object_info.last_modification_time != 0; - else - result.is_last_modified_known = false; - } - - /// GetObjectAttributes returns per-part sizes; convert to cumulative start offsets. - if (!object_info.part_sizes.empty()) - { - result.part_offsets.reserve(object_info.part_sizes.size()); - uint64_t offset = 0; - for (size_t part_size : object_info.part_sizes) - { - result.part_offsets.push_back(offset); - offset += part_size; - } - } - - if (result.part_offsets.empty() && !preserved_part_offsets.empty() && result.etag == preserved_etag) - result.part_offsets = std::move(preserved_part_offsets); + result.is_last_modified_known = object_info.last_modification_time != 0; if (identity_cache) { @@ -1162,9 +1112,7 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool identity.etag = result.etag; identity.size = result.size_bytes; identity.is_size_known = result.is_size_known; - identity.part_offsets = result.part_offsets; - identity.last_modified_observed = object_info.last_modification_time_observed; - identity.is_last_modified_known = identity.last_modified_observed && result.is_last_modified_known; + identity.is_last_modified_known = result.is_last_modified_known; if (identity.is_last_modified_known) identity.last_modified = static_cast(result.last_modified.epochTime()); identity_cache->setIfEpoch(identity_key, std::move(identity), identity_read); diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h index bd96c35946cc4..990ce86aa769e 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h @@ -144,23 +144,11 @@ class S3ObjectStorage : public IObjectStorage void tagObjects(const StoredObjects & objects, const std::string & tag_key, const std::string & tag_value) override; - ObjectMetadata getObjectMetadata(const std::string & path, bool with_tags) const override - { - return getObjectMetadata(path, with_tags, /*fetch_part_offsets=*/ false, /*use_identity_cache=*/ true); - } - ObjectMetadata getObjectMetadata(const std::string & path, bool with_tags, bool fetch_part_offsets) const override - { - return getObjectMetadata(path, with_tags, fetch_part_offsets, /*use_identity_cache=*/ true); - } - - ObjectMetadata getObjectMetadata(const RelativePathWithMetadata & object, bool with_tags, bool fetch_part_offsets) const override - { - return getObjectMetadata(object.getPath(), with_tags, fetch_part_offsets, /*use_identity_cache=*/ true); - } + ObjectMetadata getObjectMetadata(const std::string & path, bool with_tags) const override; - ObjectMetadata getObjectMetadataIgnoringIdentityCache(const RelativePathWithMetadata & object, bool with_tags, bool fetch_part_offsets) const override + ObjectMetadata getObjectMetadataIgnoringIdentityCache(const RelativePathWithMetadata & object, bool with_tags) const override { - return getObjectMetadata(object.getPath(), with_tags, fetch_part_offsets, /*use_identity_cache=*/ false); + return getObjectMetadata(object.getPath(), with_tags, /*use_identity_cache=*/ false); } std::optional tryGetObjectMetadata(const std::string & path, bool with_tags) const override; @@ -267,7 +255,7 @@ class S3ObjectStorage : public IObjectStorage /// `use_identity_cache` false still stores the result. It only skips the lookup, so a content cache /// can key itself on an ETag observed by this request. - ObjectMetadata getObjectMetadata(const std::string & path, bool with_tags, bool fetch_part_offsets, bool use_identity_cache) const; + ObjectMetadata getObjectMetadata(const std::string & path, bool with_tags, bool use_identity_cache) const; /// Runs `fn` and, if it failed because the vended credentials expired, refreshes this disk's /// client and runs it once more. `fn` must re-read the client itself, so the second run signs diff --git a/src/IO/ReadSettings.h b/src/IO/ReadSettings.h index 177ed6ef35808..09af30956faeb 100644 --- a/src/IO/ReadSettings.h +++ b/src/IO/ReadSettings.h @@ -154,10 +154,6 @@ struct ReadSettings bool use_page_cache_for_object_storage = false; PageCacheSettings page_cache_settings; - /// Fetch S3 multipart part offsets (via GetObjectAttributes) when reading object metadata, for - /// part-aligned reads. Off = cheaper HEAD (size + etag only). See Settings docs. - bool object_storage_identity_cache_fetch_part_offsets = false; - /// Experimental pipeline read executor. When `enabled`, `ReadPipeline::build` sends supported reads /// through `ReaderExecutor` instead of the legacy stack of read buffers. The long-connection knobs /// apply only on the executor path. `use_long_connections` reuses one held source connection across diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp index 5c86c168ede85..e28948af5097c 100644 --- a/src/IO/S3/Client.cpp +++ b/src/IO/S3/Client.cpp @@ -553,12 +553,6 @@ Model::GetBucketVersioningOutcome Client::GetBucketVersioning(GetBucketVersionin doRequest(request, [this](const Model::GetBucketVersioningRequest & req) { return GetBucketVersioning(req); })); } -Model::GetObjectAttributesOutcome Client::GetObjectAttributes(GetObjectAttributesRequest & request) const -{ - return processRequestResult( - doRequest(request, [this](const Model::GetObjectAttributesRequest & req) { return GetObjectAttributes(req); })); -} - Model::ListObjectsV2Outcome Client::ListObjectsV2(ListObjectsV2Request & request) const { return doRequestWithRetryNetworkErrors( diff --git a/src/IO/S3/Client.h b/src/IO/S3/Client.h index 7dfc15b406809..8290eb0a61ead 100644 --- a/src/IO/S3/Client.h +++ b/src/IO/S3/Client.h @@ -217,7 +217,6 @@ class Client : private Aws::S3::S3Client Model::ListObjectsV2Outcome ListObjectsV2(ListObjectsV2Request & request) const; Model::ListObjectsOutcome ListObjects(ListObjectsRequest & request) const; Model::GetObjectOutcome GetObject(GetObjectRequest & request) const; - Model::GetObjectAttributesOutcome GetObjectAttributes(GetObjectAttributesRequest & request) const; Model::AbortMultipartUploadOutcome AbortMultipartUpload(AbortMultipartUploadRequest & request) const; Model::CreateMultipartUploadOutcome CreateMultipartUpload(CreateMultipartUploadRequest & request) const; @@ -297,7 +296,6 @@ class Client : private Aws::S3::S3Client using Aws::S3::S3Client::ListObjectsV2; using Aws::S3::S3Client::ListObjects; using Aws::S3::S3Client::GetObject; - using Aws::S3::S3Client::GetObjectAttributes; using Aws::S3::S3Client::AbortMultipartUpload; using Aws::S3::S3Client::CreateMultipartUpload; diff --git a/src/IO/S3/Requests.h b/src/IO/S3/Requests.h index 5430281ebe765..1a86fbd9b8959 100644 --- a/src/IO/S3/Requests.h +++ b/src/IO/S3/Requests.h @@ -12,7 +12,6 @@ #include #include #include -#include #include #include #include @@ -185,7 +184,6 @@ class HeadObjectRequest: public ExtendedRequest using ListObjectsV2Request = ExtendedRequest; using ListObjectsRequest = ExtendedRequest; using GetObjectRequest = ExtendedRequest; -using GetObjectAttributesRequest = ExtendedRequest; using GetObjectTaggingRequest = ExtendedRequest; using GetBucketVersioningRequest = ExtendedRequest; diff --git a/src/IO/S3/getObjectInfo.cpp b/src/IO/S3/getObjectInfo.cpp index fcb216547f31e..23803e98b7fb1 100644 --- a/src/IO/S3/getObjectInfo.cpp +++ b/src/IO/S3/getObjectInfo.cpp @@ -5,17 +5,13 @@ #if USE_AWS_S3 -#include - namespace ProfileEvents { extern const Event S3GetObjectTagging; extern const Event S3HeadObject; extern const Event S3HeadObjectMicroseconds; - extern const Event S3GetObjectAttributes; extern const Event DiskS3GetObjectTagging; extern const Event DiskS3HeadObject; - extern const Event DiskS3GetObjectAttributes; } @@ -52,43 +48,6 @@ namespace return client.HeadObject(req); } - Aws::S3::Model::GetObjectAttributesOutcome getObjectAttributes( - const S3::Client & client, - const String & bucket, - const String & key, - const String & version_id) - { - ProfileEvents::increment(ProfileEvents::S3GetObjectAttributes); - if (client.isClientForDisk()) - ProfileEvents::increment(ProfileEvents::DiskS3GetObjectAttributes); - - S3::GetObjectAttributesRequest req; - req.SetBucket(bucket); - req.SetKey(key); - if (!version_id.empty()) - req.SetVersionId(version_id); - req.SetObjectAttributes({ - Aws::S3::Model::ObjectAttributes::ETag, - Aws::S3::Model::ObjectAttributes::ObjectSize, - Aws::S3::Model::ObjectAttributes::ObjectParts}); - - return client.GetObjectAttributes(req); - } - - /// `HeadObject` and `PutObject` spell an ETag as `"abc"` (multipart: `"abc-2"`). `GetObjectAttributes` - /// returns that value without the quotes. `If-Match` and the identity cache compare the bytes, so - /// both paths store the quoted spelling. A weak validator (`W/"abc"`) is already in HTTP form. - String quoteETag(String s) - { - if (s.empty()) - return s; - if (s.size() >= 2 && s.front() == '"' && s.back() == '"') - return s; - if (s.starts_with("W/\"") && s.ends_with('"')) - return s; - return "\"" + s + "\""; - } - Aws::S3::Model::GetObjectTaggingOutcome getObjectTagging( const S3::Client & client, const String & bucket, @@ -128,7 +87,6 @@ namespace object_info.size = static_cast(result.GetContentLength()); object_info.is_size_known = result.ContentLengthHasBeenSet(); object_info.last_modification_time = result.GetLastModified().Seconds(); - object_info.last_modification_time_observed = true; object_info.etag = result.GetETag(); if (with_metadata) @@ -236,35 +194,6 @@ ObjectInfo getObjectInfo( getAuthenticationErrorHint(error.GetErrorType())); } -ObjectInfo getObjectIdentity( - const S3::Client & client, - const String & bucket, - const String & key, - const String & version_id) -{ - { - Expect404ResponseScope scope; // a not-found here just falls through to HEAD below - auto outcome = getObjectAttributes(client, bucket, key, version_id); - if (outcome.IsSuccess()) - { - const auto & result = outcome.GetResult(); - ObjectInfo object_info; - object_info.size = static_cast(result.GetObjectSize()); - object_info.is_size_known = true; - object_info.etag = quoteETag(result.GetETag()); - for (const auto & part : result.GetObjectParts().GetParts()) - object_info.part_sizes.push_back(static_cast(part.GetSize())); - return object_info; - } - /// GetObjectAttributes unsupported (many S3-compatible stores), denied (distinct IAM action), - /// or otherwise failed - fall back to a plain HEAD. Best-effort: we lose part_sizes only. - } - - ObjectInfo object_info = getObjectInfo(client, bucket, key, version_id, /*with_metadata=*/ false, /*with_tags=*/ false); - object_info.etag = quoteETag(object_info.etag); - return object_info; -} - size_t getObjectSize( const S3::Client & client, const String & bucket, diff --git a/src/IO/S3/getObjectInfo.h b/src/IO/S3/getObjectInfo.h index 5fa8c17d15294..d3a31260f473c 100644 --- a/src/IO/S3/getObjectInfo.h +++ b/src/IO/S3/getObjectInfo.h @@ -3,7 +3,6 @@ #include "config.h" #if USE_AWS_S3 -#include #include #include #include @@ -18,17 +17,9 @@ struct ObjectInfo /// Checks if Content-Length was present in the HEAD response and we can rely on its result bool is_size_known = true; time_t last_modification_time = 0; - /// False when the response did not include Last-Modified. `GetObjectAttributes` in this SDK cannot - /// request it, so a successful attributes call leaves the time at 0 without having observed it. - /// A HEAD that returns epoch 0 sets this true: the object really has no modification time. - bool last_modification_time_observed = false; String etag; ObjectAttributes tags; // Set only if getObjectInfo() is called with `with_tags = true` ObjectAttributes metadata = {}; /// Set only if getObjectInfo() is called with `with_metadata = true`. - /// Sizes of the object's multipart-upload parts (offsets = prefix sums). Set only by - /// getObjectIdentity() when the object was multipart-uploaded and GetObjectAttributes succeeded; - /// empty for single-PUT objects or when the call fell back to HEAD. - std::vector part_sizes; }; /// Ignore if object does not exist @@ -54,18 +45,6 @@ ObjectInfo getObjectInfo( bool with_metadata = false, bool with_tags = false); -/// Fetches size + etag (+ multipart part sizes) in a single GetObjectAttributes request instead of a -/// HEAD, so a caller that also wants the multipart layout gets it without a second round-trip. Falls -/// back to getObjectInfo() (HEAD) if GetObjectAttributes is unsupported, denied, or otherwise errors, -/// so it is safe against S3-compatible stores that lack the API. The etag is the quoted spelling -/// `HeadObject` returns, including when GetObjectAttributes omits the quotes. `part_sizes` is -/// populated only on the GetObjectAttributes path for multipart objects; it is empty on the fallback path. -ObjectInfo getObjectIdentity( - const S3::Client & client, - const String & bucket, - const String & key, - const String & version_id = {}); - ObjectAttributes getObjectTags( const S3::Client & client, const String & bucket, diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 845a6d94a7a3a..a4fec01479d1d 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -391,7 +391,6 @@ namespace Setting extern const SettingsBool use_page_cache_for_disks_without_file_cache; extern const SettingsBool use_page_cache_for_local_disks; extern const SettingsBool use_page_cache_for_object_storage; - extern const SettingsBool object_storage_identity_cache_fetch_part_offsets; extern const SettingsBool use_page_cache_with_distributed_cache; extern const SettingsUInt64 use_structure_from_insertion_table_in_table_functions; extern const SettingsString workload; @@ -620,7 +619,7 @@ struct ContextSharedPart : boost::noncopyable mutable ResourceManagerPtr resource_manager; mutable UncompressedCachePtr uncompressed_cache TSA_GUARDED_BY(mutex); /// The cache of decompressed blocks. mutable MarkCachePtr mark_cache TSA_GUARDED_BY(mutex); /// Cache of marks in compressed files. - mutable ObjectStorageIdentityCachePtr object_storage_identity_cache TSA_GUARDED_BY(mutex); /// Cache of object-storage identity (size/etag/part-offsets) to avoid per-open HEADs. + mutable ObjectStorageIdentityCachePtr object_storage_identity_cache TSA_GUARDED_BY(mutex); /// Cache of object-storage identity (size/etag/Last-Modified) to avoid per-open HEADs. mutable UniqueKeyIndexCachePtr unique_key_index_cache TSA_GUARDED_BY(mutex); /// RocksDB-compatible block cache over CacheBase for the UNIQUE KEY index (nullptr when RocksDB unavailable or disabled). mutable DeleteBitmapCachePtr delete_bitmap_cache TSA_GUARDED_BY(mutex); /// UNIQUE KEY per-part delete-bitmap cache. mutable PrimaryIndexCachePtr primary_index_cache TSA_GUARDED_BY(mutex); @@ -9050,7 +9049,6 @@ ReadSettings Context::getReadSettings() const res.use_page_cache_with_distributed_cache = settings_ref[Setting::use_page_cache_with_distributed_cache]; res.use_page_cache_for_local_disks = settings_ref[Setting::use_page_cache_for_local_disks]; res.use_page_cache_for_object_storage = settings_ref[Setting::use_page_cache_for_object_storage]; - res.object_storage_identity_cache_fetch_part_offsets = settings_ref[Setting::object_storage_identity_cache_fetch_part_offsets]; res.reader_executor.enabled = settings_ref[Setting::use_reader_executor]; res.reader_executor.use_long_connections = settings_ref[Setting::reader_executor_use_long_connections]; res.reader_executor.window_size = settings_ref[Setting::reader_executor_window_size]; diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 94f971e71b975..f745bc39b1fc6 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -1571,7 +1571,7 @@ class Context: public ContextData, public std::enable_shared_from_this void clearMarkCache() const; ThreadPool & getLoadMarksThreadpool() const; - /// Object-storage identity cache (per-object size/ETag/multipart-part-offsets) to avoid a HEAD + /// Object-storage identity cache (per-object size/ETag/Last-Modified) to avoid a HEAD /// on every object open. A max_size_in_bytes of 0 leaves the cache unset (disabled). void setObjectStorageIdentityCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio); void updateObjectStorageIdentityCacheConfiguration(const Poco::Util::AbstractConfiguration & config, size_t max_cache_size); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.cpp index ea3cb97eeb645..14c3050cee5e0 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.cpp @@ -100,7 +100,7 @@ IcebergDataObjectInfo::IcebergDataObjectInfo( , resolved_storage(std::move(resolved_storage_)) { chassert(resolved_key_.empty() == (resolved_storage == nullptr)); - /// Object identity (size / etag / multipart part offsets) is intentionally not pre-populated from + /// Object identity (size / etag / Last-Modified) is intentionally not pre-populated from /// the manifest. S3ObjectStorage::getObjectMetadata loads it once and keeps it in /// ObjectStorageIdentityCache, so Iceberg data files get the real ETag used by the filesystem, /// page, and Parquet metadata caches. Pre-populating metadata here would skip that path. diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index 7a5b85fa5a3a3..827c251d6448e 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -220,7 +220,6 @@ namespace Setting extern const SettingsBool use_object_storage_list_objects_cache; extern const SettingsBool allow_experimental_iceberg_read_optimization; extern const SettingsBool azure_validate_etag_on_read; - extern const SettingsBool object_storage_identity_cache_fetch_part_offsets; } namespace @@ -459,7 +458,6 @@ std::shared_ptr StorageObjectStorageSource::createFileIterator( { const bool is_archive = configuration->isArchive(); const bool match_web_paths_only = configuration->getType() == ObjectStorageType::Web; - const bool fetch_part_offsets = local_context->getSettingsRef()[Setting::object_storage_identity_cache_fetch_part_offsets]; if (distributed_processing) { @@ -503,7 +501,7 @@ std::shared_ptr StorageObjectStorageSource::createFileIterator( iterator = std::make_unique( paths, object_storage, virtual_columns, is_archive ? nullptr : read_keys, query_settings.ignore_non_existent_file, skip_object_metadata, with_tags, - file_progress_callback, fetch_part_offsets); + file_progress_callback); } else if (reading_path.hasGlobs()) { @@ -590,8 +588,7 @@ std::shared_ptr StorageObjectStorageSource::createFileIterator( query_settings.ignore_non_existent_file, skip_object_metadata, with_tags, - file_progress_callback, - fetch_part_offsets); + file_progress_callback); } } else if (configuration->supportsFileIterator()) @@ -674,7 +671,7 @@ std::shared_ptr StorageObjectStorageSource::createFileIterator( iterator = std::make_unique( paths, object_storage, virtual_columns, is_archive ? nullptr : read_keys, query_settings.ignore_non_existent_file, /*skip_object_metadata=*/false, with_tags, - file_progress_callback, fetch_part_offsets); + file_progress_callback); } if (is_archive) @@ -1162,9 +1159,7 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade } else { - object_info->setObjectMetadata(storage_to_use->getObjectMetadata( - metadata_object, with_tags, - context_->getSettingsRef()[Setting::object_storage_identity_cache_fetch_part_offsets])); + object_info->setObjectMetadata(storage_to_use->getObjectMetadata(metadata_object, with_tags)); } } @@ -1993,9 +1988,7 @@ std::unique_ptr createReadBuffer( if (!object_info.metadata) { - object_info.metadata = object_storage->getObjectMetadata( - object_info, /*with_tags=*/ false, - /*fetch_part_offsets=*/ effective_read_settings.object_storage_identity_cache_fetch_part_offsets); + object_info.metadata = object_storage->getObjectMetadata(object_info, /*with_tags=*/ false); } else if (!object_info.metadata->is_fetched && validate_etag_on_read) { @@ -2009,17 +2002,13 @@ std::unique_ptr createReadBuffer( /// that the object was replaced. Re-read the identity before using its ETag as the cache key. /// An unchanged object keeps the same key. A replaced object gets a new key and is read again. /// The parquet flag is only a footer-only count: other Parquet reads still check the ETag on GET. - /// This refresh is a HEAD. `GetObjectAttributes` does not return Last-Modified, so using it here - /// would make this read's `_time` NULL and replace the cached time with "not observed". Part offsets - /// already stored for the same ETag are kept by `setIfEpoch`. if ((use_filesystem_cache || use_page_cache || revalidate_identity_for_parquet_metadata_cache) && object_info.metadata && object_info.metadata->is_from_identity_cache && object_info.metadata->is_size_known && object_info.metadata->isEtagUsableAsCacheKey()) { - object_info.metadata = object_storage->getObjectMetadataIgnoringIdentityCache( - object_info, /*with_tags=*/ false, /*fetch_part_offsets=*/ false); + object_info.metadata = object_storage->getObjectMetadataIgnoringIdentityCache(object_info, /*with_tags=*/ false); } if (use_page_cache && !object_info.metadata->isEtagUsableAsCacheKey()) @@ -2414,8 +2403,7 @@ StorageObjectStorageSource::KeysIterator::KeysIterator( bool ignore_non_existent_files_, bool skip_object_metadata_, bool with_tags_, - std::function file_progress_callback_, - bool fetch_part_offsets_) + std::function file_progress_callback_) : object_storage(object_storage_) , virtual_columns(virtual_columns_) , file_progress_callback(file_progress_callback_) @@ -2423,7 +2411,6 @@ StorageObjectStorageSource::KeysIterator::KeysIterator( , ignore_non_existent_files(ignore_non_existent_files_) , skip_object_metadata(skip_object_metadata_) , with_tags(with_tags_) - , fetch_part_offsets(fetch_part_offsets_) { if (read_keys_) { @@ -2457,7 +2444,7 @@ ObjectInfoPtr StorageObjectStorageSource::KeysIterator::next(size_t /* processor object_metadata = *metadata; } else - object_metadata = object_storage->getObjectMetadata(key, with_tags, fetch_part_offsets); + object_metadata = object_storage->getObjectMetadata(key, with_tags); } else { diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.h b/src/Storages/ObjectStorage/StorageObjectStorageSource.h index cacc32ac13651..196beb0e6bbf8 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.h +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.h @@ -314,8 +314,7 @@ class StorageObjectStorageSource::KeysIterator : public IObjectIterator bool ignore_non_existent_files_, bool skip_object_metadata_, bool with_tags_, - std::function file_progress_callback = {}, - bool fetch_part_offsets_ = false); + std::function file_progress_callback = {}); ~KeysIterator() override = default; @@ -332,7 +331,6 @@ class StorageObjectStorageSource::KeysIterator : public IObjectIterator const bool ignore_non_existent_files; const bool skip_object_metadata; const bool with_tags; - const bool fetch_part_offsets; }; /* diff --git a/tests/queries/0_stateless/05484_object_storage_identity_cache.sh b/tests/queries/0_stateless/05484_object_storage_identity_cache.sh index a62a23b197922..ad9283993f230 100755 --- a/tests/queries/0_stateless/05484_object_storage_identity_cache.sh +++ b/tests/queries/0_stateless/05484_object_storage_identity_cache.sh @@ -4,12 +4,10 @@ # Tag no-parallel: SYSTEM DROP OBJECT STORAGE IDENTITY CACHE clears the process-global cache # Tag no-parallel-replicas: per-query S3 request counts change when a read is spread across replicas -# The object-storage identity cache stores size and etag (and multipart part offsets, when -# requested) so a later open of the same key does not send HEAD or GetObjectAttributes. -# A cold read misses and issues one metadata request. The next read of that key is a hit -# and issues none. SYSTEM DROP OBJECT STORAGE IDENTITY CACHE removes the entry, so the -# following read misses again. object_storage_identity_cache_fetch_part_offsets switches -# the cold request from HEAD to GetObjectAttributes; a hit still skips both. +# The object-storage identity cache stores size, etag and Last-Modified so a later open of the same +# key does not send HEAD. A cold read misses and issues one HEAD. The next read of that key is a hit +# and issues none. SYSTEM DROP OBJECT STORAGE IDENTITY CACHE removes the entry, so the following read +# misses again. CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -24,17 +22,16 @@ fail() exit 1 } -# $1 label, $2 extra settings (may be empty), $3 head mode (eq0|ge1|any), $4 attrs mode, $5 hits mode, $6 misses mode +# $1 label, $2 extra settings (may be empty), $3 head mode (eq0|ge1|any), $4 hits mode, $5 misses mode read_and_check() { local label=$1 local extra=$2 local head_mode=$3 - local attrs_mode=$4 - local hits_mode=$5 - local misses_mode=$6 + local hits_mode=$4 + local misses_mode=$5 local qid="idcache_${label}_${CLICKHOUSE_DATABASE}" - local data settings row head attrs hits misses lists + local data settings row head hits misses lists settings="${READ_SETTINGS}" if [[ -n $extra ]]; then @@ -50,7 +47,6 @@ read_and_check() row=$(${CLICKHOUSE_CLIENT} -q " SELECT ProfileEvents['S3HeadObject'], - ProfileEvents['S3GetObjectAttributes'], ProfileEvents['ObjectStorageIdentityCacheHits'], ProfileEvents['ObjectStorageIdentityCacheMisses'], ProfileEvents['S3ListObjects'] @@ -65,9 +61,8 @@ read_and_check() FORMAT TabSeparated") || fail "${label}: query_log failed" [[ -n $row ]] || fail "${label}: no query_log row" - IFS=$'\t' read -r head attrs hits misses lists <<< "$row" + IFS=$'\t' read -r head hits misses lists <<< "$row" check_count "$label" S3HeadObject "$head" "$head_mode" - check_count "$label" S3GetObjectAttributes "$attrs" "$attrs_mode" check_count "$label" Hits "$hits" "$hits_mode" check_count "$label" Misses "$misses" "$misses_mode" [[ $lists == "0" ]] || fail "${label}: S3ListObjects=${lists}, expected 0 (row: ${row})" @@ -94,15 +89,10 @@ ${CLICKHOUSE_CLIENT} -q " # Drop whatever the INSERT cached, then prove a cold read, a hit, and that DROP really clears. ${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear failed" -read_and_check cold "" ge1 eq0 eq0 ge1 -read_and_check warm "" eq0 eq0 ge1 eq0 +read_and_check cold "" ge1 eq0 ge1 +read_and_check warm "" eq0 ge1 eq0 ${CLICKHOUSE_CLIENT} -q "SYSTEM DROP OBJECT STORAGE IDENTITY CACHE" || fail "drop failed" -read_and_check after_drop "" ge1 eq0 eq0 ge1 - -# GetObjectAttributes is the cold request when part offsets are requested. A following read is a hit. -${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear before attributes failed" -read_and_check attrs_cold "object_storage_identity_cache_fetch_part_offsets = 1" any ge1 eq0 ge1 -read_and_check attrs_warm "object_storage_identity_cache_fetch_part_offsets = 1" eq0 eq0 ge1 eq0 +read_and_check after_drop "" ge1 eq0 ge1 echo OK diff --git a/tests/queries/0_stateless/05485_object_storage_identity_cache_endpoint_key.sh b/tests/queries/0_stateless/05485_object_storage_identity_cache_endpoint_key.sh index 6a61bed1e231f..80dcb2b2401fb 100755 --- a/tests/queries/0_stateless/05485_object_storage_identity_cache_endpoint_key.sh +++ b/tests/queries/0_stateless/05485_object_storage_identity_cache_endpoint_key.sh @@ -30,7 +30,7 @@ read_and_check() local hits_mode=$4 local misses_mode=$5 local qid="idcache_ep_${label}_${CLICKHOUSE_DATABASE}" - local data row head attrs hits misses lists + local data row head hits misses lists data=$(${CLICKHOUSE_CLIENT} --query_id "$qid" -q \ "SELECT n FROM s3('${url}', 'test', 'testtest', 'TSV', 'n UInt64') SETTINGS ${READ_SETTINGS} FORMAT TabSeparated") \ @@ -41,7 +41,6 @@ read_and_check() row=$(${CLICKHOUSE_CLIENT} -q " SELECT ProfileEvents['S3HeadObject'], - ProfileEvents['S3GetObjectAttributes'], ProfileEvents['ObjectStorageIdentityCacheHits'], ProfileEvents['ObjectStorageIdentityCacheMisses'], ProfileEvents['S3ListObjects'] @@ -56,9 +55,8 @@ read_and_check() FORMAT TabSeparated") || fail "${label}: query_log failed" [[ -n $row ]] || fail "${label}: no query_log row" - IFS=$'\t' read -r head attrs hits misses lists <<< "$row" + IFS=$'\t' read -r head hits misses lists <<< "$row" check_count "$label" S3HeadObject "$head" "$head_mode" - check_count "$label" S3GetObjectAttributes "$attrs" eq0 check_count "$label" Hits "$hits" "$hits_mode" check_count "$label" Misses "$misses" "$misses_mode" [[ $lists == "0" ]] || fail "${label}: S3ListObjects=${lists}, expected 0 (row: ${row})" diff --git a/tests/queries/0_stateless/05486_object_storage_identity_cache_overwrite.sh b/tests/queries/0_stateless/05486_object_storage_identity_cache_overwrite.sh index bc417617cccb2..119bcaff8f32f 100755 --- a/tests/queries/0_stateless/05486_object_storage_identity_cache_overwrite.sh +++ b/tests/queries/0_stateless/05486_object_storage_identity_cache_overwrite.sh @@ -30,7 +30,7 @@ read_and_check() local hits_mode=$4 local misses_mode=$5 local qid="idcache_ow_${label}_${CLICKHOUSE_DATABASE}" - local data row head attrs hits misses lists + local data row head hits misses lists data=$(${CLICKHOUSE_CLIENT} --query_id "$qid" -q \ "SELECT n FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') ORDER BY n SETTINGS ${READ_SETTINGS} FORMAT TabSeparated") \ @@ -41,7 +41,6 @@ read_and_check() row=$(${CLICKHOUSE_CLIENT} -q " SELECT ProfileEvents['S3HeadObject'], - ProfileEvents['S3GetObjectAttributes'], ProfileEvents['ObjectStorageIdentityCacheHits'], ProfileEvents['ObjectStorageIdentityCacheMisses'], ProfileEvents['S3ListObjects'] @@ -56,9 +55,8 @@ read_and_check() FORMAT TabSeparated") || fail "${label}: query_log failed" [[ -n $row ]] || fail "${label}: no query_log row" - IFS=$'\t' read -r head attrs hits misses lists <<< "$row" + IFS=$'\t' read -r head hits misses lists <<< "$row" check_count "$label" S3HeadObject "$head" "$head_mode" - check_count "$label" S3GetObjectAttributes "$attrs" eq0 check_count "$label" Hits "$hits" "$hits_mode" check_count "$label" Misses "$misses" "$misses_mode" [[ $lists == "0" ]] || fail "${label}: S3ListObjects=${lists}, expected 0 (row: ${row})" diff --git a/tests/queries/0_stateless/05487_object_storage_identity_cache_etag_spelling.reference b/tests/queries/0_stateless/05487_object_storage_identity_cache_etag_spelling.reference deleted file mode 100644 index d86bac9de59ab..0000000000000 --- a/tests/queries/0_stateless/05487_object_storage_identity_cache_etag_spelling.reference +++ /dev/null @@ -1 +0,0 @@ -OK diff --git a/tests/queries/0_stateless/05487_object_storage_identity_cache_etag_spelling.sh b/tests/queries/0_stateless/05487_object_storage_identity_cache_etag_spelling.sh deleted file mode 100755 index 081ea29a33879..0000000000000 --- a/tests/queries/0_stateless/05487_object_storage_identity_cache_etag_spelling.sh +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env bash -# Tags: no-fasttest, no-parallel, no-parallel-replicas -# Tag no-fasttest: reads a file through the mock S3 server at localhost:11111 -# Tag no-parallel: SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE clears the process-global cache -# Tag no-parallel-replicas: per-query S3 request counts change when a read is spread across replicas - -# HeadObject spells an ETag as `"abc"`. GetObjectAttributes returns the same value without the -# quotes. The identity cache and If-Match compare those bytes, so both paths must store the quoted -# spelling. s3_validate_etag_on_read stays on: an unquoted cached ETag is sent as If-Match and the -# read fails even though the object did not change. `_etag` is the value stored for that read. - -CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CUR_DIR"/../shell_config.sh - -URL="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" -READ_SETTINGS="max_threads = 1, use_hive_partitioning = 0, parallel_replicas_for_cluster_engines = 0, enable_filesystem_cache = 0, use_page_cache_for_object_storage = 0, s3_validate_etag_on_read = 1, use_object_storage_list_objects_cache = 0" - -fail() -{ - echo "FAIL: $*" >&2 - exit 1 -} - -# $1 label, $2 extra settings, $3 head mode (eq0|ge1|any), $4 attrs mode, $5 hits mode, $6 misses mode -# prints the _etag on stdout -read_etag() -{ - local label=$1 - local extra=$2 - local head_mode=$3 - local attrs_mode=$4 - local hits_mode=$5 - local misses_mode=$6 - local qid="idcache_etag_${label}_${CLICKHOUSE_DATABASE}" - local settings data row n etag head attrs hits misses lists - - settings="${READ_SETTINGS}" - if [[ -n $extra ]]; then - settings="${settings}, ${extra}" - fi - - data=$(${CLICKHOUSE_CLIENT} --query_id "$qid" -q \ - "SELECT n, _etag FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') SETTINGS ${settings} FORMAT TabSeparated") \ - || fail "${label}: select failed" - IFS=$'\t' read -r n etag <<< "$data" - [[ $n == "42" ]] || fail "${label}: data '${data}'" - [[ $etag == '"'*'"' ]] || fail "${label}: etag '${etag}' is not quoted" - - ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "${label}: flush logs failed" - row=$(${CLICKHOUSE_CLIENT} -q " - SELECT - ProfileEvents['S3HeadObject'], - ProfileEvents['S3GetObjectAttributes'], - ProfileEvents['ObjectStorageIdentityCacheHits'], - ProfileEvents['ObjectStorageIdentityCacheMisses'], - ProfileEvents['S3ListObjects'] - FROM system.query_log - WHERE event_date >= yesterday() - AND event_time >= now() - 600 - AND type = 'QueryFinish' - AND query_id = '${qid}' - AND current_database = currentDatabase() - ORDER BY event_time_microseconds DESC - LIMIT 1 - FORMAT TabSeparated") || fail "${label}: query_log failed" - [[ -n $row ]] || fail "${label}: no query_log row" - - IFS=$'\t' read -r head attrs hits misses lists <<< "$row" - check_count "$label" S3HeadObject "$head" "$head_mode" - check_count "$label" S3GetObjectAttributes "$attrs" "$attrs_mode" - check_count "$label" Hits "$hits" "$hits_mode" - check_count "$label" Misses "$misses" "$misses_mode" - [[ $lists == "0" ]] || fail "${label}: S3ListObjects=${lists}, expected 0 (row: ${row})" - - printf '%s' "$etag" -} - -check_count() -{ - local label=$1 - local name=$2 - local got=$3 - local mode=$4 - case "$mode" in - eq0) [[ $got == "0" ]] || fail "${label}: ${name}=${got}, expected 0" ;; - ge1) [[ $got -ge 1 ]] || fail "${label}: ${name}=${got}, expected >= 1" ;; - any) ;; - *) fail "${label}: bad mode ${mode}" ;; - esac -} - -${CLICKHOUSE_CLIENT} -q " - INSERT INTO FUNCTION s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') - SELECT 42 - SETTINGS s3_truncate_on_insert = 1" || fail "insert failed" - -${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear failed" -head_etag=$(read_etag head "" ge1 eq0 eq0 ge1) || exit 1 - -${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear before attributes failed" -# GetObjectAttributes may fall back to HEAD, so the HEAD counter is unconstrained. -attrs_etag=$(read_etag attrs "object_storage_identity_cache_fetch_part_offsets = 1" any ge1 eq0 ge1) || exit 1 -[[ $attrs_etag == "$head_etag" ]] || fail "etag spelling differs: head '${head_etag}', attributes '${attrs_etag}'" - -# The attributes read filled the cache. The next read must reuse that ETag, not HEAD again. -hit_etag=$(read_etag attrs_hit "object_storage_identity_cache_fetch_part_offsets = 1" eq0 eq0 ge1 eq0) || exit 1 -[[ $hit_etag == "$head_etag" ]] || fail "cached etag '${hit_etag}' differs from head '${head_etag}'" - -echo OK diff --git a/tests/queries/0_stateless/05490_object_storage_identity_cache_external_replace.sh b/tests/queries/0_stateless/05490_object_storage_identity_cache_external_replace.sh index 26f604e5e95b9..4c4ab17875eff 100755 --- a/tests/queries/0_stateless/05490_object_storage_identity_cache_external_replace.sh +++ b/tests/queries/0_stateless/05490_object_storage_identity_cache_external_replace.sh @@ -30,7 +30,7 @@ read_and_check() local hits_mode=$4 local misses_mode=$5 local qid="idcache_ext_${label}_${CLICKHOUSE_DATABASE}" - local data row head attrs hits misses lists + local data row head hits misses lists data=$(${CLICKHOUSE_CLIENT} --query_id "$qid" -q \ "SELECT n FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') ORDER BY n SETTINGS ${READ_SETTINGS} FORMAT TabSeparated") \ @@ -41,7 +41,6 @@ read_and_check() row=$(${CLICKHOUSE_CLIENT} -q " SELECT ProfileEvents['S3HeadObject'], - ProfileEvents['S3GetObjectAttributes'], ProfileEvents['ObjectStorageIdentityCacheHits'], ProfileEvents['ObjectStorageIdentityCacheMisses'], ProfileEvents['S3ListObjects'] @@ -56,9 +55,8 @@ read_and_check() FORMAT TabSeparated") || fail "${label}: query_log failed" [[ -n $row ]] || fail "${label}: no query_log row" - IFS=$'\t' read -r head attrs hits misses lists <<< "$row" + IFS=$'\t' read -r head hits misses lists <<< "$row" check_count "$label" S3HeadObject "$head" "$head_mode" - check_count "$label" S3GetObjectAttributes "$attrs" eq0 check_count "$label" Hits "$hits" "$hits_mode" check_count "$label" Misses "$misses" "$misses_mode" [[ $lists == "0" ]] || fail "${label}: S3ListObjects=${lists}, expected 0 (row: ${row})" diff --git a/tests/queries/0_stateless/05495_object_storage_identity_cache_part_offsets_time.reference b/tests/queries/0_stateless/05495_object_storage_identity_cache_part_offsets_time.reference deleted file mode 100644 index d86bac9de59ab..0000000000000 --- a/tests/queries/0_stateless/05495_object_storage_identity_cache_part_offsets_time.reference +++ /dev/null @@ -1 +0,0 @@ -OK diff --git a/tests/queries/0_stateless/05495_object_storage_identity_cache_part_offsets_time.sh b/tests/queries/0_stateless/05495_object_storage_identity_cache_part_offsets_time.sh deleted file mode 100755 index 4e5c9603fa9a9..0000000000000 --- a/tests/queries/0_stateless/05495_object_storage_identity_cache_part_offsets_time.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env bash -# Tags: no-fasttest, no-parallel, no-parallel-replicas -# Tag no-fasttest: reads a file through the mock S3 server at localhost:11111 -# Tag no-parallel: the identity cache and the failpoint are process-global -# Tag no-parallel-replicas: per-query S3 request counts change when a read is spread across replicas - -# GetObjectAttributes does not return Last-Modified. Caching that as "time unknown" makes every later -# `_time` NULL. The failpoint simulates that response. The next read, with part offsets off, must HEAD -# and return the real time. - -CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CUR_DIR"/../shell_config.sh - -URL="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" -READ_SETTINGS="max_threads = 1, use_hive_partitioning = 0, parallel_replicas_for_cluster_engines = 0, enable_filesystem_cache = 0, use_page_cache_for_object_storage = 0, s3_validate_etag_on_read = 0, use_object_storage_list_objects_cache = 0" -FAILPOINT="object_storage_identity_cache_skip_last_modified" - -fail() -{ - echo "FAIL: $*" >&2 - exit 1 -} - -cleanup() -{ - ${CLICKHOUSE_CLIENT} -q "SYSTEM DISABLE FAILPOINT ${FAILPOINT}" >/dev/null 2>&1 || true -} -trap cleanup EXIT - -read_time() -{ - local label=$1 - local extra_settings=$2 - local qid="idcache_time_${label}_${CLICKHOUSE_DATABASE}" - local value - - value=$(${CLICKHOUSE_CLIENT} --query_id "$qid" -q " - SELECT if(isNull(_time), 'NULL', toString(_time)) - FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') - SETTINGS ${READ_SETTINGS}${extra_settings} - FORMAT TabSeparated") || fail "${label}: select failed" - printf '%s' "$value" -} - -head_count() -{ - local qid=$1 - ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "flush logs failed" - ${CLICKHOUSE_CLIENT} -q " - SELECT ProfileEvents['S3HeadObject'] - FROM system.query_log - WHERE event_date >= yesterday() - AND event_time >= now() - 600 - AND type = 'QueryFinish' - AND query_id = '${qid}' - AND current_database = currentDatabase() - ORDER BY event_time_microseconds DESC - LIMIT 1 - FORMAT TabSeparated" -} - -${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear failed" - -${CLICKHOUSE_CLIENT} -q " - INSERT INTO FUNCTION s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') - SELECT 1 - SETTINGS s3_truncate_on_insert = 1" || fail "insert failed" - -${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear after insert failed" -${CLICKHOUSE_CLIENT} -q "SYSTEM ENABLE FAILPOINT ${FAILPOINT}" || fail "enable failpoint failed" - -without_time=$(read_time part_offsets ", object_storage_identity_cache_fetch_part_offsets = 1") -[[ $without_time == "NULL" ]] || fail "part-offset fetch stored a time '${without_time}', failpoint did not apply" - -${CLICKHOUSE_CLIENT} -q "SYSTEM DISABLE FAILPOINT ${FAILPOINT}" || fail "disable failpoint failed" - -later_qid="idcache_time_later_${CLICKHOUSE_DATABASE}" -later=$(${CLICKHOUSE_CLIENT} --query_id "$later_qid" -q " - SELECT if(isNull(_time), 'NULL', toString(_time)) - FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') - SETTINGS ${READ_SETTINGS} - FORMAT TabSeparated") || fail "later select failed" -[[ $later != "NULL" && $later != "1970-01-01 00:00:00" && -n $later ]] || fail "later _time is '${later}'" - -heads=$(head_count "$later_qid") -[[ $heads -ge 1 ]] || fail "later read did not HEAD (S3HeadObject=${heads})" - -echo OK diff --git a/tests/queries/0_stateless/05496_object_storage_identity_cache_filesystem_cache_part_offsets_time.reference b/tests/queries/0_stateless/05496_object_storage_identity_cache_filesystem_cache_part_offsets_time.reference deleted file mode 100644 index d86bac9de59ab..0000000000000 --- a/tests/queries/0_stateless/05496_object_storage_identity_cache_filesystem_cache_part_offsets_time.reference +++ /dev/null @@ -1 +0,0 @@ -OK diff --git a/tests/queries/0_stateless/05496_object_storage_identity_cache_filesystem_cache_part_offsets_time.sh b/tests/queries/0_stateless/05496_object_storage_identity_cache_filesystem_cache_part_offsets_time.sh deleted file mode 100755 index 01c76922baddf..0000000000000 --- a/tests/queries/0_stateless/05496_object_storage_identity_cache_filesystem_cache_part_offsets_time.sh +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env bash -# Tags: no-fasttest, no-parallel, no-parallel-replicas -# Tag no-fasttest: reads a file through the mock S3 server at localhost:11111 -# Tag no-parallel: the identity cache and the filesystem cache are process-global -# Tag no-parallel-replicas: per-query S3 request counts change when a read is spread across replicas - -# A filesystem-cache revalidation must HEAD. GetObjectAttributes does not return Last-Modified, and -# storing that result makes this read's `_time` NULL and drops the time already in the identity cache. -# The mock S3 server rejects GetObjectAttributes and falls back to HEAD, so the assertion is the -# request itself: the revalidating read must not call GetObjectAttributes. - -CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CUR_DIR"/../shell_config.sh - -URL="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" -COMMON_SETTINGS="max_threads = 1, use_hive_partitioning = 0, parallel_replicas_for_cluster_engines = 0, filesystem_cache_name = 'cache_for_readbigat', use_page_cache_for_object_storage = 0, s3_validate_etag_on_read = 1, use_object_storage_list_objects_cache = 0" - -fail() -{ - echo "FAIL: $*" >&2 - exit 1 -} - -# $1 label, $2 extra settings, $3 head mode, $4 attributes mode, $5 get mode, $6 cache mode -# Prints the _time value. -read_time() -{ - local label=$1 - local extra=$2 - local head_mode=$3 - local attributes_mode=$4 - local get_mode=$5 - local cache_mode=$6 - local qid="idcache_fscache_time_${label}_${CLICKHOUSE_DATABASE}" - local value row head attributes gets cache_bytes - - value=$(${CLICKHOUSE_CLIENT} --query_id "$qid" -q " - SELECT if(isNull(_time), 'NULL', toString(_time)) - FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') - SETTINGS ${COMMON_SETTINGS}${extra} - FORMAT TabSeparated") || fail "${label}: select failed" - - ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "${label}: flush logs failed" - row=$(${CLICKHOUSE_CLIENT} -q " - SELECT - ProfileEvents['S3HeadObject'], - ProfileEvents['S3GetObjectAttributes'], - ProfileEvents['S3GetObject'], - ProfileEvents['CachedReadBufferReadFromCacheBytes'] - FROM system.query_log - WHERE event_date >= yesterday() - AND event_time >= now() - 600 - AND type = 'QueryFinish' - AND query_id = '${qid}' - AND current_database = currentDatabase() - ORDER BY event_time_microseconds DESC - LIMIT 1 - FORMAT TabSeparated") || fail "${label}: query_log failed" - [[ -n $row ]] || fail "${label}: no query_log row" - - IFS=$'\t' read -r head attributes gets cache_bytes <<< "$row" - case "$head_mode" in - eq0) [[ $head == "0" ]] || fail "${label}: S3HeadObject=${head}, expected 0" ;; - ge1) [[ $head -ge 1 ]] || fail "${label}: S3HeadObject=${head}, expected >= 1" ;; - *) fail "${label}: bad head mode ${head_mode}" ;; - esac - case "$attributes_mode" in - eq0) [[ $attributes == "0" ]] || fail "${label}: S3GetObjectAttributes=${attributes}, expected 0" ;; - *) fail "${label}: bad attributes mode ${attributes_mode}" ;; - esac - case "$get_mode" in - eq0) [[ $gets == "0" ]] || fail "${label}: S3GetObject=${gets}, expected 0" ;; - ge1) [[ $gets -ge 1 ]] || fail "${label}: S3GetObject=${gets}, expected >= 1" ;; - *) fail "${label}: bad get mode ${get_mode}" ;; - esac - case "$cache_mode" in - ge1) [[ $cache_bytes -ge 1 ]] || fail "${label}: CachedReadBufferReadFromCacheBytes=${cache_bytes}, expected >= 1" ;; - any) ;; - *) fail "${label}: bad cache mode ${cache_mode}" ;; - esac - - printf '%s' "$value" -} - -${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear identity cache failed" -${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR FILESYSTEM CACHE 'cache_for_readbigat'" || fail "clear filesystem cache failed" - -${CLICKHOUSE_CLIENT} -q " - INSERT INTO FUNCTION s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') - SELECT 1 - SETTINGS s3_truncate_on_insert = 1" || fail "insert failed" - -${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear identity cache after insert failed" - -primed=$(read_time primed ", enable_filesystem_cache = 1, object_storage_identity_cache_fetch_part_offsets = 0" ge1 eq0 ge1 any) -[[ $primed != "NULL" && $primed != "1970-01-01 00:00:00" && -n $primed ]] || fail "primed _time is '${primed}'" - -revalidated=$(read_time revalidated ", enable_filesystem_cache = 1, object_storage_identity_cache_fetch_part_offsets = 1" ge1 eq0 eq0 ge1) -[[ $revalidated == "$primed" ]] || fail "revalidated _time is '${revalidated}', primed is '${primed}'" - -later=$(read_time later ", enable_filesystem_cache = 0, object_storage_identity_cache_fetch_part_offsets = 0" eq0 eq0 ge1 any) -[[ $later == "$primed" ]] || fail "later _time is '${later}', primed is '${primed}'" - -echo OK From 1cda5892dc880760a5260ade8a9b22e4379663a6 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 17 Nov 2026 20:38:49 +0100 Subject: [PATCH 25/37] Object storage: re-HEAD an identity-cache hit before a skip decision An identity-cache hit returns the previous size and ETag. Two decisions used that metadata without a GET, so an external replace could be dropped unread: - `s3_skip_empty_files` (on by default) skipped a file whose cached size was 0, in `StorageObjectStorageSource::createReader` and `ReadBufferIterator`. - A query condition cache hit that dropped every row group, or every row group left after the bucket filter, skipped a file whose cached ETag was the key. A new helper `revalidateIdentityCachedObjectMetadata` replaces an identity-cache hit with a HEAD before those decisions. Data-lake keys do not contain the ETag, so they are not revalidated for the condition cache. Reads that do not skip still use the cache and do not pay the HEAD. Adds test 05498 that replaces an empty object and a Parquet object that the condition cache would have skipped, then expects the new rows after a HEAD. Co-authored-by: Cursor --- .../ObjectStorage/ReadBufferIterator.cpp | 6 + .../StorageObjectStorageSource.cpp | 80 ++++++++--- src/Storages/ObjectStorage/Utils.h | 4 + ...age_identity_cache_skip_replaced.reference | 1 + ...ct_storage_identity_cache_skip_replaced.sh | 133 ++++++++++++++++++ 5 files changed, 208 insertions(+), 16 deletions(-) create mode 100644 tests/queries/0_stateless/05498_object_storage_identity_cache_skip_replaced.reference create mode 100755 tests/queries/0_stateless/05498_object_storage_identity_cache_skip_replaced.sh diff --git a/src/Storages/ObjectStorage/ReadBufferIterator.cpp b/src/Storages/ObjectStorage/ReadBufferIterator.cpp index 09e01becdeb0f..1886612ebf774 100644 --- a/src/Storages/ObjectStorage/ReadBufferIterator.cpp +++ b/src/Storages/ObjectStorage/ReadBufferIterator.cpp @@ -258,6 +258,12 @@ ReadBufferIterator::Data ReadBufferIterator::next() prev_read_keys_size = read_keys.size(); } + if (query_settings.skip_empty_files && current_object_info->getObjectMetadata() + && current_object_info->getObjectMetadata()->is_from_identity_cache + && current_object_info->getObjectMetadata()->size_bytes == 0 + && current_object_info->getObjectMetadata()->is_size_known) + revalidateIdentityCachedObjectMetadata(*current_object_info, object_storage); + if (query_settings.skip_empty_files && current_object_info->getObjectMetadata() && current_object_info->getObjectMetadata()->size_bytes == 0 && current_object_info->getObjectMetadata()->is_size_known) diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index 827c251d6448e..1fac01d7b2987 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -1163,19 +1163,38 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade } } - if (query_settings.skip_empty_files && object_info->getObjectMetadata()->size_bytes == 0 + /// The cached size is the previous object's. Re-HEAD before treating it as empty, or a + /// replaced object is skipped and never read. `s3_skip_empty_files` defaults to true. + const auto cached_metadata = object_info->getObjectMetadata(); + if (cached_metadata + && cached_metadata->is_from_identity_cache + && query_settings.skip_empty_files + && cached_metadata->is_size_known + && cached_metadata->size_bytes == 0) + revalidateIdentityCachedObjectMetadata(*object_info, object_storage); + + if (query_settings.skip_empty_files && object_info->getObjectMetadata() + && object_info->getObjectMetadata()->size_bytes == 0 && object_info->getObjectMetadata()->is_size_known) continue; + bool skip_object = false; if (query_condition_cache && !object_info->file_bucket_info) { - const auto query_condition_cache_key = makeQueryConditionCacheKey(*object_info, configuration->isDataLakeConfiguration()); - std::optional matching_marks; - if (query_condition_cache_key) - matching_marks = query_condition_cache->read( - storage_id.uuid, *query_condition_cache_key, *format_filter_info->condition_hash); - if (matching_marks.has_value()) + /// A hit that drops every row group never GETs. The key contains the ETag, and an + /// identity-cache hit still has the previous one, so confirm it before dropping the file. + /// Data-lake keys do not contain the ETag. One confirmation is enough: the HEAD clears + /// `is_from_identity_cache`. + for (size_t attempt = 0; attempt < 2; ++attempt) { + const auto query_condition_cache_key = makeQueryConditionCacheKey(*object_info, configuration->isDataLakeConfiguration()); + std::optional matching_marks; + if (query_condition_cache_key) + matching_marks = query_condition_cache->read( + storage_id.uuid, *query_condition_cache_key, *format_filter_info->condition_hash); + if (!matching_marks.has_value()) + break; + const auto & marks = *matching_marks; size_t total_row_groups = marks.size(); std::vector matching_row_groups; @@ -1191,20 +1210,36 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade format_filter_info->filter_actions_dag->dumpNames(), object_info->getFileName()); - if (matching_row_groups.empty()) - continue; + bool drops_object = matching_row_groups.empty(); + FileBucketInfoPtr filtered; + if (!drops_object) + { + auto file_bucket_info = FormatFactory::instance().getFileBucketInfo( + object_info->getFileFormat().value_or(configuration->getFormat())); + if (file_bucket_info) + { + filtered = file_bucket_info->filterByMatchingRowGroups(matching_row_groups); + if (!filtered) + drops_object = true; + } + } - auto file_bucket_info = FormatFactory::instance().getFileBucketInfo( - object_info->getFileFormat().value_or(configuration->getFormat())); - if (file_bucket_info) + const auto metadata_now = object_info->getObjectMetadata(); + if (drops_object && attempt == 0 && metadata_now && metadata_now->is_from_identity_cache + && !configuration->isDataLakeConfiguration()) { - auto filtered = file_bucket_info->filterByMatchingRowGroups(matching_row_groups); - if (!filtered) - continue; - object_info->file_bucket_info = std::move(filtered); + revalidateIdentityCachedObjectMetadata(*object_info, object_storage); + continue; } + + skip_object = drops_object; + if (filtered) + object_info->file_bucket_info = std::move(filtered); + break; } } + if (skip_object) + continue; break; } @@ -1935,6 +1970,19 @@ std::future StorageObjectStorageSource return create_reader_scheduler([=, this] { return createReader(); }, Priority{}); } +void revalidateIdentityCachedObjectMetadata(ObjectInfo & object_info, const ObjectStoragePtr & object_storage) +{ + const auto metadata = object_info.getObjectMetadata(); + if (!metadata || !metadata->is_from_identity_cache) + return; + + auto metadata_object = object_info.relative_path_with_metadata; + metadata_object.relative_path = object_info.getPathOrPathToArchiveIfArchive(); + object_info.setObjectMetadata( + object_info.getResolvedStorage(object_storage)->getObjectMetadataIgnoringIdentityCache( + metadata_object, /*with_tags=*/ false)); +} + std::unique_ptr createReadBuffer( RelativePathWithMetadata & object_info, const ObjectStoragePtr & object_storage, diff --git a/src/Storages/ObjectStorage/Utils.h b/src/Storages/ObjectStorage/Utils.h index 49a8f8b05b5be..fe165198c54a5 100644 --- a/src/Storages/ObjectStorage/Utils.h +++ b/src/Storages/ObjectStorage/Utils.h @@ -31,6 +31,10 @@ void validateSupportedColumns( /// An empty column name has no identifier to render it with, so it cannot survive analysis. void validateLakeSchemaColumnNames(const NamesAndTypesList & schema, std::string_view lake_name); +/// Replace identity-cache metadata with a HEAD. A hit still has the previous size and ETag, and a caller +/// that then skips the read never issues the GET that would notice a replace. +void revalidateIdentityCachedObjectMetadata(ObjectInfo & object_info, const ObjectStoragePtr & object_storage); + std::unique_ptr createReadBuffer( RelativePathWithMetadata & object_info, const ObjectStoragePtr & object_storage, diff --git a/tests/queries/0_stateless/05498_object_storage_identity_cache_skip_replaced.reference b/tests/queries/0_stateless/05498_object_storage_identity_cache_skip_replaced.reference new file mode 100644 index 0000000000000..d86bac9de59ab --- /dev/null +++ b/tests/queries/0_stateless/05498_object_storage_identity_cache_skip_replaced.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/05498_object_storage_identity_cache_skip_replaced.sh b/tests/queries/0_stateless/05498_object_storage_identity_cache_skip_replaced.sh new file mode 100755 index 0000000000000..2990f4bad5660 --- /dev/null +++ b/tests/queries/0_stateless/05498_object_storage_identity_cache_skip_replaced.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-parallel-replicas +# Tag no-fasttest: reads a file through the mock S3 server at localhost:11111 +# Tag no-parallel: the identity cache and the query condition cache are process-global +# Tag no-parallel-replicas: per-query S3 request counts change when a read is spread across replicas + +# An identity-cache hit still has the previous size and ETag. Two decisions use that metadata and +# then do not GET: s3_skip_empty_files (default on) skips a cached size of 0, and the query condition +# cache skips a file whose cached ETag matched no row group. An external replace must still be read. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +fail() +{ + echo "FAIL: $*" >&2 + exit 1 +} + +READ_SETTINGS="max_threads = 1, use_hive_partitioning = 0, parallel_replicas_for_cluster_engines = 0, enable_filesystem_cache = 0, use_page_cache_for_object_storage = 0, s3_validate_etag_on_read = 1, use_object_storage_list_objects_cache = 0, s3_skip_empty_files = 1" + +# --- A cached empty object must not hide a later non-empty replace. --- + +EMPTY_URL="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}_empty.tsv" + +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear identity cache failed" + +curl -fsS -X PUT "$EMPTY_URL" --data-binary '' >/dev/null || fail "empty put failed" + +empty_cold=$(${CLICKHOUSE_CLIENT} -q " + SELECT n FROM s3('${EMPTY_URL}', 'test', 'testtest', 'TSV', 'n UInt64') + SETTINGS ${READ_SETTINGS} FORMAT TabSeparated") || fail "empty cold select failed" +[[ -z $empty_cold ]] || fail "empty cold: data '${empty_cold}'" + +empty_warm=$(${CLICKHOUSE_CLIENT} -q " + SELECT n FROM s3('${EMPTY_URL}', 'test', 'testtest', 'TSV', 'n UInt64') + SETTINGS ${READ_SETTINGS} FORMAT TabSeparated") || fail "empty warm select failed" +[[ -z $empty_warm ]] || fail "empty warm: data '${empty_warm}'" + +empty_payload="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}_empty.tsv" +printf '7\n' > "$empty_payload" +curl -fsS -X PUT "$EMPTY_URL" --upload-file "$empty_payload" >/dev/null || fail "empty replace put failed" + +replaced_qid="idcache_skip_empty_${CLICKHOUSE_DATABASE}" +empty_replaced=$(${CLICKHOUSE_CLIENT} --query_id "$replaced_qid" -q " + SELECT n FROM s3('${EMPTY_URL}', 'test', 'testtest', 'TSV', 'n UInt64') + SETTINGS ${READ_SETTINGS} FORMAT TabSeparated") || fail "empty replaced select failed" +[[ $empty_replaced == "7" ]] || fail "empty replaced: data '${empty_replaced}', expected 7" + +${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "flush logs failed" +empty_heads=$(${CLICKHOUSE_CLIENT} -q " + SELECT ProfileEvents['S3HeadObject'] + FROM system.query_log + WHERE event_date >= yesterday() + AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND query_id = '${replaced_qid}' + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC + LIMIT 1 + FORMAT TabSeparated") || fail "empty query_log failed" +[[ $empty_heads -ge 1 ]] || fail "empty replaced did not HEAD (S3HeadObject=${empty_heads})" + +# --- A query-condition-cache hit on the previous ETag must not drop the replaced file. --- + +PARQUET_URL="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}_qcc.parquet" +TABLE="idcache_skip_qcc_${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear identity cache before parquet failed" +${CLICKHOUSE_CLIENT} -q "SYSTEM DROP QUERY CONDITION CACHE" || fail "drop query condition cache failed" +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ${TABLE}" || fail "drop table failed" + +${CLICKHOUSE_CLIENT} -q " + CREATE TABLE ${TABLE} (n UInt64) + ENGINE = S3('${PARQUET_URL}', 'test', 'testtest', 'Parquet')" || fail "create table failed" + +${CLICKHOUSE_CLIENT} -q " + INSERT INTO ${TABLE} SELECT toUInt64(1) + SETTINGS s3_truncate_on_insert = 1" || fail "parquet insert failed" + +FILTER_SETTINGS="${READ_SETTINGS}, use_query_condition_cache = 1" + +qcc_first=$(${CLICKHOUSE_CLIENT} -q " + SELECT count() FROM ${TABLE} WHERE n = 3 SETTINGS ${FILTER_SETTINGS}") || fail "qcc first select failed" +[[ $qcc_first == "0" ]] || fail "qcc first: '${qcc_first}', expected 0" + +# The second read must hit the condition cache, or the replace below does not exercise the skip. +qcc_second_qid="idcache_skip_qcc_warm_${CLICKHOUSE_DATABASE}" +qcc_second=$(${CLICKHOUSE_CLIENT} --query_id "$qcc_second_qid" -q " + SELECT count() FROM ${TABLE} WHERE n = 3 SETTINGS ${FILTER_SETTINGS}") || fail "qcc warm select failed" +[[ $qcc_second == "0" ]] || fail "qcc warm: '${qcc_second}', expected 0" + +${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "flush logs before qcc hit check failed" +qcc_hits=$(${CLICKHOUSE_CLIENT} -q " + SELECT ProfileEvents['QueryConditionCacheHits'] + FROM system.query_log + WHERE event_date >= yesterday() + AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND query_id = '${qcc_second_qid}' + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC + LIMIT 1 + FORMAT TabSeparated") || fail "qcc warm query_log failed" +[[ $qcc_hits -ge 1 ]] || fail "qcc warm was not a condition-cache hit (QueryConditionCacheHits=${qcc_hits})" + +parquet_payload="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}_qcc.parquet" +${CLICKHOUSE_CLIENT} -q "SELECT toUInt64(3) AS n FORMAT Parquet" > "$parquet_payload" || fail "local parquet failed" +curl -fsS -X PUT "$PARQUET_URL" --upload-file "$parquet_payload" >/dev/null || fail "parquet replace put failed" + +qcc_replaced_qid="idcache_skip_qcc_replaced_${CLICKHOUSE_DATABASE}" +qcc_replaced=$(${CLICKHOUSE_CLIENT} --query_id "$qcc_replaced_qid" -q " + SELECT count() FROM ${TABLE} WHERE n = 3 SETTINGS ${FILTER_SETTINGS}") || fail "qcc replaced select failed" +[[ $qcc_replaced == "1" ]] || fail "qcc replaced: '${qcc_replaced}', expected 1" + +${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "flush logs after qcc replace failed" +qcc_heads=$(${CLICKHOUSE_CLIENT} -q " + SELECT ProfileEvents['S3HeadObject'] + FROM system.query_log + WHERE event_date >= yesterday() + AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND query_id = '${qcc_replaced_qid}' + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC + LIMIT 1 + FORMAT TabSeparated") || fail "qcc replaced query_log failed" +[[ $qcc_heads -ge 1 ]] || fail "qcc replaced did not HEAD (S3HeadObject=${qcc_heads})" + +${CLICKHOUSE_CLIENT} -q "DROP TABLE ${TABLE}" || fail "drop table at end failed" + +echo OK From e05a9bcf5fc517d9493a9010a8a072626fdec458 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 17 Nov 2026 20:44:12 +0100 Subject: [PATCH 26/37] Object storage: re-HEAD before skipping an empty condition-cache replacement The empty-file check in createReader runs once, before the query condition cache loop. A condition-cache hit that drops the whole file re-HEADs the object, which can reveal that the replacement is empty. That path then fell through and opened the object, and formats such as Parquet throw on an empty file. Apply the empty-file skip again after the re-HEAD, honoring `s3_skip_empty_files`. Extend the 05498 test with an empty Parquet replacement behind a condition-cache hit. Co-authored-by: Cursor --- .../StorageObjectStorageSource.cpp | 11 ++++ ...ct_storage_identity_cache_skip_replaced.sh | 64 +++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index 1fac01d7b2987..c9d9e89ed4413 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -1229,6 +1229,17 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade && !configuration->isDataLakeConfiguration()) { revalidateIdentityCachedObjectMetadata(*object_info, object_storage); + + /// The empty-file check above already ran, against the previous size. A replaced + /// object can be empty, and reading it throws for formats such as Parquet. + /// `s3_skip_empty_files` defaults to true. + const auto refreshed = object_info->getObjectMetadata(); + if (query_settings.skip_empty_files && refreshed + && refreshed->is_size_known && refreshed->size_bytes == 0) + { + skip_object = true; + break; + } continue; } diff --git a/tests/queries/0_stateless/05498_object_storage_identity_cache_skip_replaced.sh b/tests/queries/0_stateless/05498_object_storage_identity_cache_skip_replaced.sh index 2990f4bad5660..836216defedba 100755 --- a/tests/queries/0_stateless/05498_object_storage_identity_cache_skip_replaced.sh +++ b/tests/queries/0_stateless/05498_object_storage_identity_cache_skip_replaced.sh @@ -7,6 +7,7 @@ # An identity-cache hit still has the previous size and ETag. Two decisions use that metadata and # then do not GET: s3_skip_empty_files (default on) skips a cached size of 0, and the query condition # cache skips a file whose cached ETag matched no row group. An external replace must still be read. +# A condition-cache re-HEAD that finds an empty replacement must skip it, not open it. CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -130,4 +131,67 @@ qcc_heads=$(${CLICKHOUSE_CLIENT} -q " ${CLICKHOUSE_CLIENT} -q "DROP TABLE ${TABLE}" || fail "drop table at end failed" +# --- A condition-cache re-HEAD that finds an empty replacement must skip it, not open it. --- + +EMPTY_PARQUET_URL="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}_qcc_empty.parquet" +EMPTY_TABLE="idcache_skip_qcc_empty_${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear identity cache before empty parquet failed" +${CLICKHOUSE_CLIENT} -q "SYSTEM DROP QUERY CONDITION CACHE" || fail "drop query condition cache before empty parquet failed" +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ${EMPTY_TABLE}" || fail "drop empty parquet table failed" + +${CLICKHOUSE_CLIENT} -q " + CREATE TABLE ${EMPTY_TABLE} (n UInt64) + ENGINE = S3('${EMPTY_PARQUET_URL}', 'test', 'testtest', 'Parquet')" || fail "create empty parquet table failed" + +${CLICKHOUSE_CLIENT} -q " + INSERT INTO ${EMPTY_TABLE} SELECT toUInt64(1) + SETTINGS s3_truncate_on_insert = 1" || fail "empty parquet insert failed" + +empty_qcc_first=$(${CLICKHOUSE_CLIENT} -q " + SELECT count() FROM ${EMPTY_TABLE} WHERE n = 3 SETTINGS ${FILTER_SETTINGS}") || fail "empty parquet first select failed" +[[ $empty_qcc_first == "0" ]] || fail "empty parquet first: '${empty_qcc_first}', expected 0" + +empty_qcc_warm_qid="idcache_skip_qcc_empty_warm_${CLICKHOUSE_DATABASE}" +empty_qcc_warm=$(${CLICKHOUSE_CLIENT} --query_id "$empty_qcc_warm_qid" -q " + SELECT count() FROM ${EMPTY_TABLE} WHERE n = 3 SETTINGS ${FILTER_SETTINGS}") || fail "empty parquet warm select failed" +[[ $empty_qcc_warm == "0" ]] || fail "empty parquet warm: '${empty_qcc_warm}', expected 0" + +${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "flush logs before empty parquet hit check failed" +empty_qcc_hits=$(${CLICKHOUSE_CLIENT} -q " + SELECT ProfileEvents['QueryConditionCacheHits'] + FROM system.query_log + WHERE event_date >= yesterday() + AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND query_id = '${empty_qcc_warm_qid}' + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC + LIMIT 1 + FORMAT TabSeparated") || fail "empty parquet warm query_log failed" +[[ $empty_qcc_hits -ge 1 ]] || fail "empty parquet warm was not a condition-cache hit (QueryConditionCacheHits=${empty_qcc_hits})" + +curl -fsS -X PUT "$EMPTY_PARQUET_URL" --data-binary '' >/dev/null || fail "empty parquet replace put failed" + +empty_qcc_replaced_qid="idcache_skip_qcc_empty_replaced_${CLICKHOUSE_DATABASE}" +empty_qcc_replaced=$(${CLICKHOUSE_CLIENT} --query_id "$empty_qcc_replaced_qid" -q " + SELECT count() FROM ${EMPTY_TABLE} WHERE n = 3 SETTINGS ${FILTER_SETTINGS}") || fail "empty parquet replaced select failed" +[[ $empty_qcc_replaced == "0" ]] || fail "empty parquet replaced: '${empty_qcc_replaced}', expected 0" + +${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "flush logs after empty parquet replace failed" +empty_qcc_heads=$(${CLICKHOUSE_CLIENT} -q " + SELECT ProfileEvents['S3HeadObject'] + FROM system.query_log + WHERE event_date >= yesterday() + AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND query_id = '${empty_qcc_replaced_qid}' + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC + LIMIT 1 + FORMAT TabSeparated") || fail "empty parquet replaced query_log failed" +[[ $empty_qcc_heads -ge 1 ]] || fail "empty parquet replaced did not HEAD (S3HeadObject=${empty_qcc_heads})" + +${CLICKHOUSE_CLIENT} -q "DROP TABLE ${EMPTY_TABLE}" || fail "drop empty parquet table at end failed" + echo OK From d208648c6b86d97c77591f41eb9f5d65f42e3c06 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 17 Nov 2026 21:04:36 +0100 Subject: [PATCH 27/37] Object storage: rename the private S3 metadata overload to fix overloaded-virtual The private three-argument `S3ObjectStorage::getObjectMetadata` shared its name with the `IObjectStorage::getObjectMetadata` overloads, so clang's `-Woverloaded-virtual` (an error under `-Werror`) reported it as hiding the two-argument `RelativePathWithMetadata` overload. Rename it to `getObjectMetadataImpl`; the public overrides keep their signatures. Co-authored-by: Cursor --- .../DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp | 4 ++-- .../DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp index b7ca4f6f76952..a53860e25a481 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp @@ -1019,10 +1019,10 @@ std::optional S3ObjectStorage::tryGetObjectMetadataImpl( ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool with_tags) const { - return getObjectMetadata(path, with_tags, /*use_identity_cache=*/ true); + return getObjectMetadataImpl(path, with_tags, /*use_identity_cache=*/ true); } -ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool with_tags, bool use_identity_cache) const +ObjectMetadata S3ObjectStorage::getObjectMetadataImpl(const std::string & path, bool with_tags, bool use_identity_cache) const { /// When the caller only needs identity (size / etag / Last-Modified, not tags or user metadata) we /// take a fast path: consult the identity cache to skip the HEAD on repeat opens. A miss issues a diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h index 483a215c68393..972b213cc5b64 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h @@ -149,7 +149,7 @@ class S3ObjectStorage : public IObjectStorage ObjectMetadata getObjectMetadataIgnoringIdentityCache(const RelativePathWithMetadata & object, bool with_tags) const override { - return getObjectMetadata(object.getPath(), with_tags, /*use_identity_cache=*/ false); + return getObjectMetadataImpl(object.getPath(), with_tags, /*use_identity_cache=*/ false); } std::optional tryGetObjectMetadata(const std::string & path, bool with_tags) const override; @@ -258,7 +258,7 @@ class S3ObjectStorage : public IObjectStorage /// `use_identity_cache` false still stores the result. It only skips the lookup, so a content cache /// can key itself on an ETag observed by this request. - ObjectMetadata getObjectMetadata(const std::string & path, bool with_tags, bool use_identity_cache) const; + ObjectMetadata getObjectMetadataImpl(const std::string & path, bool with_tags, bool use_identity_cache) const; /// The single-attempt client for the object-creating requests of a write that asks for it, else null. std::shared_ptr objectCreationClient(const WriteSettings & write_settings) const; From be68b12bbb56379850fc31360e8e66d0448c486f Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 17 Nov 2026 21:35:20 +0100 Subject: [PATCH 28/37] Object storage: pin populate-on-miss in the identity cache filesystem cache test The stateless runner can randomize `read_from_filesystem_cache_if_exists_otherwise_bypass_cache` to 1. In that mode a cache miss reads from S3 and does not store the segment. Test 05494 clears `cache_for_readbigat`, so the cold read leaves nothing behind, and the cached read misses and issues a `S3GetObject`. The test expects zero GETs there. Set the setting to 0 in the test's read settings, as other filesystem cache tests do, so a miss populates the cache and the warm read is a cache hit. Co-authored-by: Cursor --- ...object_storage_identity_cache_filesystem_cache_replace.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/05494_object_storage_identity_cache_filesystem_cache_replace.sh b/tests/queries/0_stateless/05494_object_storage_identity_cache_filesystem_cache_replace.sh index 7383e29a0cd9c..7e4883a5dd7b8 100755 --- a/tests/queries/0_stateless/05494_object_storage_identity_cache_filesystem_cache_replace.sh +++ b/tests/queries/0_stateless/05494_object_storage_identity_cache_filesystem_cache_replace.sh @@ -13,7 +13,10 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CUR_DIR"/../shell_config.sh URL="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" -READ_SETTINGS="max_threads = 1, use_hive_partitioning = 0, parallel_replicas_for_cluster_engines = 0, enable_filesystem_cache = 1, filesystem_cache_name = 'cache_for_readbigat', use_page_cache_for_object_storage = 0, s3_validate_etag_on_read = 1, use_object_storage_list_objects_cache = 0" +# Pin populate-on-miss. The stateless runner can set +# `read_from_filesystem_cache_if_exists_otherwise_bypass_cache` to 1, and then a miss is read from +# S3 and not stored, so the second read still issues a GET. +READ_SETTINGS="max_threads = 1, use_hive_partitioning = 0, parallel_replicas_for_cluster_engines = 0, enable_filesystem_cache = 1, filesystem_cache_name = 'cache_for_readbigat', read_from_filesystem_cache_if_exists_otherwise_bypass_cache = 0, use_page_cache_for_object_storage = 0, s3_validate_etag_on_read = 1, use_object_storage_list_objects_cache = 0" fail() { From 96dba9e1659020b0ac9a10d36e123523dd4fec5b Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Wed, 18 Nov 2026 09:59:10 +0100 Subject: [PATCH 29/37] Object storage: clear the identity cache in tests that rewrite objects outside ClickHouse The identity cache keeps the ETag of an object. A test that replaces an object in MinIO without going through ClickHouse leaves a stale ETag behind. The next read sends it as If-Match, the server returns 412, and the query fails with `S3_OBJECT_CHANGED_DURING_READ` (surfaced as `INCORRECT_DATA` for Iceberg manifests). - `test_database_iceberg`: clear the identity cache after the manifest rewrites in `test_select_manifest_without_sequence_number` and `test_select_manifest_without_snapshot_id`. - `test_iceberg_inverted_manifest_bounds`: clear the identity cache after each `fput_object` rewrite of the manifests. - `test_storage_gcp_auth`: a warm read hits the identity cache and skips HEAD, so one GET adds one token request. Expect 3 token requests instead of 4 after the second read. - `04033_s3_brace_glob_no_extra_head`: tag `no-parallel` and clear the identity cache first. The test counts HEAD requests, and a hit from an earlier test would make the count 0. - `05491_url_object_metadata_shard`: the test server answers Range requests with 206 and Content-Range. A 200 response triggers `Unable to read with range header`, and the stateless runner fails a test on any client stderr. Integration shard failures (4/8 and 2/8 of amd_asan_ubsan, db disk, old analyzer): https://altinity-build-artifacts.s3.amazonaws.com/json.html?PR=2888&sha=0ffd9d6cc9a390df868d26bc5496fa564e54a908&name_0=PR&name_1=Integration%20tests%20%28amd_asan_ubsan%2C%20db%20disk%2C%20old%20analyzer%2C%204%2F8%29 https://altinity-build-artifacts.s3.amazonaws.com/json.html?PR=2888&sha=0ffd9d6cc9a390df868d26bc5496fa564e54a908&name_0=PR&name_1=Integration%20tests%20%28amd_asan_ubsan%2C%20db%20disk%2C%20old%20analyzer%2C%202%2F8%29 Stateless tests: https://altinity-build-artifacts.s3.amazonaws.com/json.html?PR=2888&sha=0ffd9d6cc9a390df868d26bc5496fa564e54a908&name_0=PR&name_1=Stateless%20tests%20%28amd_asan_ubsan%2C%20cas%20s3%20storage%2C%20parallel%2C%201%2F2%29 PR: https://github.com/Altinity/ClickHouse/pull/2888 Co-authored-by: Cursor --- .../integration/test_database_iceberg/test.py | 6 +++++ .../integration/test_storage_gcp_auth/test.py | 4 ++- .../test_iceberg_inverted_manifest_bounds.py | 4 +++ .../04033_s3_brace_glob_no_extra_head.sql | 6 ++++- .../05491_url_object_metadata_shard.sh | 25 ++++++++++++++++++- 5 files changed, 42 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_database_iceberg/test.py b/tests/integration/test_database_iceberg/test.py index 98d41bad325a2..5011ddbb9c78d 100644 --- a/tests/integration/test_database_iceberg/test.py +++ b/tests/integration/test_database_iceberg/test.py @@ -1137,6 +1137,9 @@ def test_select_manifest_without_sequence_number(started_cluster, rewrite_manife ["sequence_number", "min_sequence_number", "content"], ) + # The rewrite goes straight to MinIO. The identity cache still holds the old ETag, and the next + # GET would send it as If-Match and fail. + node.query("SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE") node.query("SYSTEM DROP ICEBERG METADATA CACHE") assert num_rows == int(node.query(f"SELECT count() FROM (SELECT * FROM {table_ref})")) @@ -1183,6 +1186,9 @@ def test_select_manifest_without_snapshot_id(started_cluster, rewrite_manifest_l ["sequence_number", "min_sequence_number", "content"], ) + # The rewrite goes straight to MinIO. The identity cache still holds the old ETag, and the next + # GET would send it as If-Match and fail. + node.query("SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE") node.query("SYSTEM DROP ICEBERG METADATA CACHE") assert num_rows == int(node.query(f"SELECT count() FROM (SELECT * FROM {table_ref})")) diff --git a/tests/integration/test_storage_gcp_auth/test.py b/tests/integration/test_storage_gcp_auth/test.py index f898ce417ea70..f57ffe9be7453 100644 --- a/tests/integration/test_storage_gcp_auth/test.py +++ b/tests/integration/test_storage_gcp_auth/test.py @@ -131,7 +131,9 @@ def get_num_requests(): assert get_num_requests() == 2 assert node.query("SELECT * FROM s3_table") == "OK\n" - assert get_num_requests() == 4 + # The first read stored this object's identity, so this read does not HEAD. One GET fetches one + # token. A cold read would HEAD and GET and add two. + assert get_num_requests() == 3 assert ( node.query( "SELECT * FROM s3(gcs_conn, filename='test.txt', format='LineAsString')" diff --git a/tests/integration/test_storage_iceberg_no_spark/test_iceberg_inverted_manifest_bounds.py b/tests/integration/test_storage_iceberg_no_spark/test_iceberg_inverted_manifest_bounds.py index e752d00a60520..e86a7072f4107 100644 --- a/tests/integration/test_storage_iceberg_no_spark/test_iceberg_inverted_manifest_bounds.py +++ b/tests/integration/test_storage_iceberg_no_spark/test_iceberg_inverted_manifest_bounds.py @@ -348,6 +348,9 @@ def pruned_files(where): # them, while `id` is still well formed in the same manifest entry and must keep pruning. manifests = MinioManifests(cluster.minio_client, cluster.minio_bucket, key_prefix) assert _swap_bounds_in_manifests(manifests, only_field_id=2) > 0 + # `fput_object` replaces the manifest outside ClickHouse. The identity cache still holds the + # previous ETag, and the next GET sends it as `If-Match`. + instance.query("SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE") assert pruned_files(d_expr) == 0 assert pruned_files(id_expr) == 1 @@ -358,5 +361,6 @@ def pruned_files(where): e_expr = "e < toDecimal32(0.5, 1)" assert pruned_files(e_expr) == 1 assert _set_bounds_in_manifests(manifests, 3, b"\x7f\xff\xff\xf0", b"\x7f\xff\xff\xfa") > 0 + instance.query("SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE") assert pruned_files(e_expr) == 0 assert pruned_files(id_expr) == 1 diff --git a/tests/queries/0_stateless/04033_s3_brace_glob_no_extra_head.sql b/tests/queries/0_stateless/04033_s3_brace_glob_no_extra_head.sql index 8698b4ab874c6..68c1f9deba297 100644 --- a/tests/queries/0_stateless/04033_s3_brace_glob_no_extra_head.sql +++ b/tests/queries/0_stateless/04033_s3_brace_glob_no_extra_head.sql @@ -1,5 +1,6 @@ --- Tags: no-fasttest, no-parallel-replicas +-- Tags: no-fasttest, no-parallel, no-parallel-replicas -- Tag no-fasttest: Depends on AWS +-- Tag no-parallel: the identity cache is process-global, so another test can HEAD these keys first -- Tag no-parallel-replicas: test counts exact S3 API calls per query, which differs under parallel replicas -- Verify that reading from S3 with brace-expansion globs like {a,b,c}.tsv @@ -9,6 +10,9 @@ SET max_threads = 1; SET use_hive_partitioning=1; +-- Drop identities stored by earlier tests. A hit does not send HEAD, so the count below would be 0. +SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE; + SELECT * FROM s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3; diff --git a/tests/queries/0_stateless/05491_url_object_metadata_shard.sh b/tests/queries/0_stateless/05491_url_object_metadata_shard.sh index 889c23190766f..308f9fdee0292 100755 --- a/tests/queries/0_stateless/05491_url_object_metadata_shard.sh +++ b/tests/queries/0_stateless/05491_url_object_metadata_shard.sh @@ -54,9 +54,32 @@ def make_handler(body): self.send_response(404) self.end_headers() return - self.send_response(200) + # A known size makes url() send Range: bytes=0-(size-1). A 200 for that request is logged + # as a warning, and the stateless runner fails the test on any client stderr. + total = len(data) + start, end, ranged = 0, total - 1, False + range_header = self.headers.get('Range') if path == '/data/file.tsv' else None + if range_header and total: + spec = range_header.split('=', 1)[-1].split(',')[0].strip() + left, _, right = spec.partition('-') + if left: + start = int(left) + if right: + end = int(right) + if start < 0 or end < start or start >= total: + self.send_response(416) + self.send_header('Content-Range', 'bytes */' + str(total)) + self.end_headers() + return + end = min(end, total - 1) + data = data[start:end + 1] + ranged = True + self.send_response(206 if ranged else 200) self.send_header('Content-Type', content_type) + self.send_header('Accept-Ranges', 'bytes') self.send_header('Content-Length', str(len(data))) + if ranged: + self.send_header('Content-Range', 'bytes ' + str(start) + '-' + str(end) + '/' + str(total)) self.end_headers() if with_body: self.wfile.write(data) From 9f77d4e86b6a0aefe749d29c415daa9e2ac070ac Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Wed, 18 Nov 2026 18:38:56 +0100 Subject: [PATCH 30/37] Key the object storage identity cache by credentials The identity cache reused an object's size, ETag and Last-Modified across S3 principals. A read signed with a different secret therefore skipped the HEAD, and the failing GET returned the signature error instead of `HTTP response code: 403`, which `test_sts_smoke_env_credentials_disabled_no_opt_in` checks. Store a SipHash of the access key, secret and session token in each entry, and hit only when it matches. A different principal misses and sends its own HEAD. An entry learned by another principal is left in place. Related: https://github.com/Altinity/ClickHouse/pull/2888 Co-authored-by: Cursor --- .../ObjectStorageIdentityCache.cpp | 4 +-- .../ObjectStorageIdentityCache.h | 9 +++++-- .../ObjectStorages/S3/S3ObjectStorage.cpp | 25 ++++++++++++++++++- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp index 3e6beb932fc2d..a51f1f73be0dc 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp @@ -29,9 +29,9 @@ ObjectStorageIdentityCache::ObjectStorageIdentityCache(const String & cache_poli { } -std::optional ObjectStorageIdentityCache::tryGet(const String & key) +std::optional ObjectStorageIdentityCache::tryGet(const String & key, UInt64 credential_hash) { - if (auto mapped = Base::get(key)) + if (auto mapped = Base::get(key); mapped && mapped->credential_hash == credential_hash) { ProfileEvents::increment(ProfileEvents::ObjectStorageIdentityCacheHits); return *mapped; diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h index bed0ed4c20642..f9b4ae3b2a1f3 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h @@ -29,6 +29,10 @@ struct ObjectStorageIdentity /// Orders stores of this key. A fetch that started earlier must not overwrite one that started later. /// Not part of the object's identity. UInt64 fetch_seq = 0; + /// SipHash of the access key, secret, and session token that signed the HEAD. Not a secret. + /// A hit is only valid for the same principal: the STS mock used in tests returns one access key + /// with a different secret per role session, so the access key alone does not separate them. + UInt64 credential_hash = 0; size_t memoryUsage() const { @@ -83,8 +87,9 @@ class ObjectStorageIdentityCache public: ObjectStorageIdentityCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio); - /// Look up an identity by `makeObjectStorageIdentityCacheKey`. Increments the hit/miss ProfileEvents. - std::optional tryGet(const String & key); + /// Look up an identity by `makeObjectStorageIdentityCacheKey` for the principal `credential_hash`. + /// An entry learned by another principal is a miss and is left in place. Increments the hit/miss ProfileEvents. + std::optional tryGet(const String & key, UInt64 credential_hash); /// Epoch at the start of a metadata fetch, and a sequence that orders that fetch against others. struct Read diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp index 3d12fd89a695f..8ff697e2a94f5 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp @@ -35,6 +35,7 @@ #include #include +#include #include #include #include @@ -69,6 +70,23 @@ namespace DB::FailPoints namespace DB { + +/// Distinguishes principals that sign with the same access key and a different secret or session token. +static UInt64 s3CredentialHash(const S3::Client & client) +{ + const auto credentials = client.getCredentials(); + SipHash hash; + const auto update_field = [&](const auto & field) + { + hash.update(field.data(), field.size()); + hash.update("\0", 1); + }; + update_field(credentials.GetAWSAccessKeyId()); + update_field(credentials.GetAWSSecretKey()); + update_field(credentials.GetSessionToken()); + return hash.get64(); +} + namespace Setting { extern const SettingsBool s3_validate_request_settings; @@ -1054,7 +1072,10 @@ ObjectMetadata S3ObjectStorage::getObjectMetadataImpl(const std::string & path, if (use_identity_cache && identity_cache) { - if (auto cached = identity_cache->tryGet(identity_key)) + /// Computed only for a lookup. `getCredentials` resolves an assumed role, and a different + /// secret must not reuse this entry. The STS test mock returns one access key for every session. + const UInt64 credential_hash = s3CredentialHash(*client.get()); + if (auto cached = identity_cache->tryGet(identity_key, credential_hash)) { ObjectMetadata result; result.size_bytes = cached->size; @@ -1140,6 +1161,8 @@ ObjectMetadata S3ObjectStorage::getObjectMetadataImpl(const std::string & path, identity.is_last_modified_known = result.is_last_modified_known; if (identity.is_last_modified_known) identity.last_modified = static_cast(result.last_modified.epochTime()); + /// After a credential refresh the client that signed the HEAD may not be the one looked up. + identity.credential_hash = s3CredentialHash(*client.get()); identity_cache->setIfEpoch(identity_key, std::move(identity), identity_read); } From 0c385e1cd247939404300976213406f64066babc Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Thu, 19 Nov 2026 12:59:52 +0100 Subject: [PATCH 31/37] Object storage: do not use a cached identity for reads without ETag validation With `s3_validate_etag_on_read = 0` (including `compatibility` before 26.7) the GET carries no `If-Match` and its response ETag is not compared. A cached size and ETag then stay in use after another client replaces the object: - A small file is prefetched up to `getFileSize()`, the cached size, so a larger replacement is returned as the previous prefix, and the next query does the same while the entry stays cached. - Parquet, ORC, and Arrow use the cached size as the file length. `S3ObjectStorage::getObjectMetadataImpl` now skips the identity-cache lookup when the query context has `s3_validate_etag_on_read = 0`. It still HEADs and stores the result, so a later read with validation on can hit. `StorageObjectStorageSource::createReadBuffer` also re-HEADs a cache hit when the reading context does not validate the ETag, in case the hit was taken on a thread without that setting. Test `05499_object_storage_identity_cache_unvalidated_replace` replaces a cached object externally and checks that a read with validation off returns the new rows with a HEAD, and that the next read with validation on is a cache hit. PR: https://github.com/Altinity/ClickHouse/pull/2888 Co-authored-by: Cursor --- .../ObjectStorages/S3/S3ObjectStorage.cpp | 17 +++- .../StorageObjectStorageSource.cpp | 13 ++- ...entity_cache_unvalidated_replace.reference | 1 + ...rage_identity_cache_unvalidated_replace.sh | 79 +++++++++++++++++++ 4 files changed, 105 insertions(+), 5 deletions(-) create mode 100644 tests/queries/0_stateless/05499_object_storage_identity_cache_unvalidated_replace.reference create mode 100755 tests/queries/0_stateless/05499_object_storage_identity_cache_unvalidated_replace.sh diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp index 8ff697e2a94f5..adf2fe46e1c2c 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp @@ -1070,7 +1070,22 @@ ObjectMetadata S3ObjectStorage::getObjectMetadataImpl(const std::string & path, if (auto global_context = Context::getGlobalContextInstance()) identity_cache = global_context->getObjectStorageIdentityCache(); - if (use_identity_cache && identity_cache) + /// A cached size is the right bound of a small-file prefetch and the file length for Parquet. + /// That is safe when the GET is pinned to the cached ETag. With `s3_validate_etag_on_read` off + /// (including `compatibility` before 26.7) nothing checks the ETag, so an external replace would + /// be read as the previous generation. Skip the hit and HEAD. The result is still stored, so a + /// later read that does check the ETag can hit. + bool consult_identity_cache = use_identity_cache; + if (consult_identity_cache) + { + if (auto query_context = CurrentThread::tryGetQueryContext()) + { + if (!query_context->getSettingsRef()[Setting::s3_validate_etag_on_read]) + consult_identity_cache = false; + } + } + + if (consult_identity_cache && identity_cache) { /// Computed only for a lookup. `getCredentials` resolves an assumed role, and a different /// secret must not reuse this entry. The STS test mock returns one access key for every session. diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index c9d9e89ed4413..bf72ca5aff661 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -2061,11 +2061,16 @@ std::unique_ptr createReadBuffer( /// that the object was replaced. Re-read the identity before using its ETag as the cache key. /// An unchanged object keeps the same key. A replaced object gets a new key and is read again. /// The parquet flag is only a footer-only count: other Parquet reads still check the ETag on GET. - if ((use_filesystem_cache || use_page_cache || revalidate_identity_for_parquet_metadata_cache) - && object_info.metadata - && object_info.metadata->is_from_identity_cache + /// + /// The same re-read is required when this query does not check the ETag at all. `getObjectMetadata` + /// skips the hit when the query context carries `s3_validate_etag_on_read = 0`, but a hit taken + /// without that context still has the previous size, and a small-file prefetch stops at that size. + const bool identity_from_cache = object_info.metadata && object_info.metadata->is_from_identity_cache; + const bool revalidate_for_content_cache = identity_from_cache + && (use_filesystem_cache || use_page_cache || revalidate_identity_for_parquet_metadata_cache) && object_info.metadata->is_size_known - && object_info.metadata->isEtagUsableAsCacheKey()) + && object_info.metadata->isEtagUsableAsCacheKey(); + if (revalidate_for_content_cache || (identity_from_cache && !validate_etag_on_read)) { object_info.metadata = object_storage->getObjectMetadataIgnoringIdentityCache(object_info, /*with_tags=*/ false); } diff --git a/tests/queries/0_stateless/05499_object_storage_identity_cache_unvalidated_replace.reference b/tests/queries/0_stateless/05499_object_storage_identity_cache_unvalidated_replace.reference new file mode 100644 index 0000000000000..d86bac9de59ab --- /dev/null +++ b/tests/queries/0_stateless/05499_object_storage_identity_cache_unvalidated_replace.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/05499_object_storage_identity_cache_unvalidated_replace.sh b/tests/queries/0_stateless/05499_object_storage_identity_cache_unvalidated_replace.sh new file mode 100755 index 0000000000000..c19a1e5c4c340 --- /dev/null +++ b/tests/queries/0_stateless/05499_object_storage_identity_cache_unvalidated_replace.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-parallel-replicas +# Tag no-fasttest: reads a file through the mock S3 server at localhost:11111 +# Tag no-parallel: the identity cache is process-global, and another test may clear it +# Tag no-parallel-replicas: per-query S3 request counts change when a read is spread across replicas + +# `s3_validate_etag_on_read = 0` sends the GET with no If-Match. A cached size must not stay the read +# bound: a small file is prefetched up to that size, so a larger external replace would be returned as +# the old prefix. The read HEADs again and returns the new rows. The HEAD also refreshes the entry, so +# the next read with validation on is a hit. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +URL="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" +COMMON_SETTINGS="max_threads = 1, use_hive_partitioning = 0, parallel_replicas_for_cluster_engines = 0, enable_filesystem_cache = 0, use_page_cache_for_object_storage = 0, use_object_storage_list_objects_cache = 0" +VALIDATED_SETTINGS="${COMMON_SETTINGS}, s3_validate_etag_on_read = 1" +UNVALIDATED_SETTINGS="${COMMON_SETTINGS}, s3_validate_etag_on_read = 0" + +fail() +{ + echo "FAIL: $*" >&2 + exit 1 +} + +# $1 label, $2 settings, $3 expected rows, $4 head mode (eq0 or ge1) +read_and_check() +{ + local label=$1 + local settings=$2 + local expected=$3 + local head_mode=$4 + local qid="idcache_unvalidated_${label}_${CLICKHOUSE_DATABASE}" + local data row head + + data=$(${CLICKHOUSE_CLIENT} --query_id "$qid" -q \ + "SELECT n FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') ORDER BY n SETTINGS ${settings} FORMAT TabSeparated") \ + || fail "${label}: select failed" + [[ $data == "$expected" ]] || fail "${label}: data '${data}', expected '${expected}'" + + ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "${label}: flush logs failed" + row=$(${CLICKHOUSE_CLIENT} -q " + SELECT ProfileEvents['S3HeadObject'] + FROM system.query_log + WHERE event_date >= yesterday() + AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND query_id = '${qid}' + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC + LIMIT 1 + FORMAT TabSeparated") || fail "${label}: query_log failed" + [[ -n $row ]] || fail "${label}: no query_log row" + head=$row + case "$head_mode" in + eq0) [[ $head == "0" ]] || fail "${label}: S3HeadObject=${head}, expected 0" ;; + ge1) [[ $head -ge 1 ]] || fail "${label}: S3HeadObject=${head}, expected >= 1" ;; + *) fail "${label}: bad mode ${head_mode}" ;; + esac +} + +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear failed" + +payload="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" +printf '1\n' > "$payload" +curl -fsS -X PUT "$URL" --upload-file "$payload" >/dev/null || fail "first put failed" + +read_and_check cold "$VALIDATED_SETTINGS" $'1' ge1 +read_and_check warm "$VALIDATED_SETTINGS" $'1' eq0 + +# This PUT does not go through ClickHouse, so the cached identity still says the object is one row. +printf '10\n20\n30\n' > "$payload" +curl -fsS -X PUT "$URL" --upload-file "$payload" >/dev/null || fail "replace put failed" + +read_and_check replaced "$UNVALIDATED_SETTINGS" $'10\n20\n30' ge1 +read_and_check refreshed "$VALIDATED_SETTINGS" $'10\n20\n30' eq0 + +echo OK From 2608efaaab0453ef1023effe92801402bd2ca7c8 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Thu, 19 Nov 2026 13:42:52 +0100 Subject: [PATCH 32/37] Object storage: drop the identity cache entry for backup writes and deletes `BackupWriterS3` writes, copies, and deletes objects through `WriteBufferFromS3`, `copyS3File`, `copyS3FileRange`, `copyDataToS3File`, and `deleteFileFromS3` directly. Those paths do not go through `S3ObjectStorage`, so they never dropped the entry in `ObjectStorageIdentityCache`. A key that a backup overwrote kept its previous ETag. The next read with `s3_validate_etag_on_read = 1` sent that ETag in `If-Match`, got a 412, and failed with `S3_OBJECT_CHANGED_DURING_READ` until `rejectCachedIdentity` dropped the entry. `BackupWriterS3` now invalidates the key after each mutation: - `writeFile` and `writeFileIfNotExists` through the `WriteBufferFromS3` completion callback, which also runs when the post-upload size check fails - `copyFileFromDisk`, `copyFile`, and `copyDataToFile` after the copy returns - `removeFile` and `removeFiles` after the delete returns Test `05500_object_storage_identity_cache_backup_replace` caches a backup object, writes a second backup to the same URL, and checks that the next read succeeds and issues a `HEAD`. PR: https://github.com/Altinity/ClickHouse/pull/2888 Co-authored-by: Cursor --- src/Backups/BackupIO_S3.cpp | 72 ++++++++++++++++--- ...ge_identity_cache_backup_replace.reference | 1 + ...t_storage_identity_cache_backup_replace.sh | 72 +++++++++++++++++++ 3 files changed, 134 insertions(+), 11 deletions(-) create mode 100644 tests/queries/0_stateless/05500_object_storage_identity_cache_backup_replace.reference create mode 100755 tests/queries/0_stateless/05500_object_storage_identity_cache_backup_replace.sh diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp index de465d12106ef..bfe936871fabd 100644 --- a/src/Backups/BackupIO_S3.cpp +++ b/src/Backups/BackupIO_S3.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include @@ -357,6 +358,26 @@ class S3BackupClientCreator return res; } + + /// Drop the identity cached for `key`. Backup IO writes and deletes through `WriteBufferFromS3`, + /// `copyS3File`, and `deleteFileFromS3`, not through `S3ObjectStorage`, so nothing else drops the entry. + /// The next read HEADs instead of sending `If-Match` for the previous ETag. + void invalidateObjectStorageIdentityCache(const String & endpoint, const String & bucket, const String & key) + { + auto global_context = Context::getGlobalContextInstance(); + if (!global_context) + { + return; + } + + auto cache = global_context->getObjectStorageIdentityCache(); + if (!cache) + { + return; + } + + cache->invalidate(makeObjectStorageIdentityCacheKey(endpoint, bucket, key)); + } } @@ -649,6 +670,7 @@ void BackupWriterS3::copyFileFromDisk( const size_t source_size = ca_plan ? ca_plan->object.bytes_size : (copy_encrypted ? src_disk->getEncryptedFileSize(src_path) : src_disk->getFileSize(src_path)); const bool whole_object = (src_offset == 0) && (length == source_size); + const String dest_key = fs::path(s3_uri.key) / path_in_backup; if (whole_object) { @@ -660,7 +682,7 @@ void BackupWriterS3::copyFileFromDisk( length, /* dest_s3_client */ client, /* dest_bucket */ s3_uri.bucket, - /* dest_key */ fs::path(s3_uri.key) / path_in_backup, + /* dest_key */ dest_key, s3_settings.request_settings, read_settings, blob_storage_log, @@ -678,7 +700,7 @@ void BackupWriterS3::copyFileFromDisk( /* src_object_size= */ source_size, /* dest_s3_client */ client, /* dest_bucket */ s3_uri.bucket, - /* dest_key */ fs::path(s3_uri.key) / path_in_backup, + /* dest_key */ dest_key, s3_settings.request_settings, read_settings, blob_storage_log, @@ -686,6 +708,8 @@ void BackupWriterS3::copyFileFromDisk( create_read_buffer); } + /// The copy has replaced `dest_key`. A read that cached the previous ETag must not send it as `If-Match`. + invalidateObjectStorageIdentityCache(s3_uri.endpoint, s3_uri.bucket, dest_key); return; /// copied! } } @@ -699,6 +723,7 @@ void BackupWriterS3::copyFile(const String & destination, const String & source, LOG_TRACE(log, "Copying file inside backup from {} to {}", source, destination); const auto source_key = fs::path(s3_uri.key) / source; + const String dest_key = fs::path(s3_uri.key) / destination; copyS3File( client, /* src_bucket */ s3_uri.bucket, @@ -706,7 +731,7 @@ void BackupWriterS3::copyFile(const String & destination, const String & source, size, /* dest_s3_client= */ client, /* dest_bucket= */ s3_uri.bucket, - /* dest_key= */ fs::path(s3_uri.key) / destination, + /* dest_key= */ dest_key, s3_settings.request_settings, read_settings, blob_storage_log, @@ -717,13 +742,16 @@ void BackupWriterS3::copyFile(const String & destination, const String & source, return std::make_unique( client, s3_uri.bucket, source_key, s3_uri.version_id, s3_settings.request_settings, read_settings); }); + invalidateObjectStorageIdentityCache(s3_uri.endpoint, s3_uri.bucket, dest_key); } void BackupWriterS3::copyDataToFile(const String & path_in_backup, const CreateReadBufferFunction & create_read_buffer, UInt64 start_pos, UInt64 length) { - copyDataToS3File(create_read_buffer, start_pos, length, client, s3_uri.bucket, fs::path(s3_uri.key) / path_in_backup, + const String dest_key = fs::path(s3_uri.key) / path_in_backup; + copyDataToS3File(create_read_buffer, start_pos, length, client, s3_uri.bucket, dest_key, s3_settings.request_settings, blob_storage_log, threadPoolCallbackRunnerUnsafe(getBackupsIOThreadPool().get(), ThreadName::S3_BACKUP_WRITER)); + invalidateObjectStorageIdentityCache(s3_uri.endpoint, s3_uri.bucket, dest_key); } BackupWriterS3::~BackupWriterS3() = default; @@ -752,38 +780,57 @@ std::unique_ptr BackupWriterS3::readFile(const String & file_name, s std::unique_ptr BackupWriterS3::writeFile(const String & file_name) { + /// Captured by value. The buffer can outlive this writer, and the cache itself is process-global. + const String endpoint = s3_uri.endpoint; + const String bucket = s3_uri.bucket; + const String object_key = fs::path(s3_uri.key) / file_name; return std::make_unique( client, - s3_uri.bucket, - fs::path(s3_uri.key) / file_name, + bucket, + object_key, DBMS_DEFAULT_BUFFER_SIZE, s3_settings.request_settings, blob_storage_log, std::nullopt, threadPoolCallbackRunnerUnsafe(getBackupsIOThreadPool().get(), ThreadName::S3_BACKUP_WRITER), - write_settings); + write_settings, + /* object_creation_client */ nullptr, + [endpoint, bucket, object_key]() + { + invalidateObjectStorageIdentityCache(endpoint, bucket, object_key); + }); } std::unique_ptr BackupWriterS3::writeFileIfNotExists(const String & file_name) { WriteSettings conditional_write_settings = write_settings; conditional_write_settings.object_storage_write_if_none_match = "*"; + const String endpoint = s3_uri.endpoint; + const String bucket = s3_uri.bucket; + const String object_key = fs::path(s3_uri.key) / file_name; return std::make_unique( client, - s3_uri.bucket, - fs::path(s3_uri.key) / file_name, + bucket, + object_key, DBMS_DEFAULT_BUFFER_SIZE, s3_settings.request_settings, blob_storage_log, std::nullopt, threadPoolCallbackRunnerUnsafe(getBackupsIOThreadPool().get(), ThreadName::S3_BACKUP_WRITER), - conditional_write_settings); + conditional_write_settings, + /* object_creation_client */ nullptr, + [endpoint, bucket, object_key]() + { + invalidateObjectStorageIdentityCache(endpoint, bucket, object_key); + }); } void BackupWriterS3::removeFile(const String & file_name) { - deleteFileFromS3(client, s3_uri.bucket, fs::path(s3_uri.key) / file_name, /* if_exists = */ true, + const String object_key = fs::path(s3_uri.key) / file_name; + deleteFileFromS3(client, s3_uri.bucket, object_key, /* if_exists = */ true, blob_storage_log); + invalidateObjectStorageIdentityCache(s3_uri.endpoint, s3_uri.bucket, object_key); } void BackupWriterS3::removeFiles(const Strings & file_names) @@ -798,6 +845,9 @@ void BackupWriterS3::removeFiles(const Strings & file_names) deleteFilesFromS3(client, s3_uri.bucket, keys, /* if_exists = */ true, s3_capabilities, batch_size, blob_storage_log); + /// A key that was deleted, or was already gone, must not keep its previous size and ETag. + for (const auto & key : keys) + invalidateObjectStorageIdentityCache(s3_uri.endpoint, s3_uri.bucket, key); } } diff --git a/tests/queries/0_stateless/05500_object_storage_identity_cache_backup_replace.reference b/tests/queries/0_stateless/05500_object_storage_identity_cache_backup_replace.reference new file mode 100644 index 0000000000000..d86bac9de59ab --- /dev/null +++ b/tests/queries/0_stateless/05500_object_storage_identity_cache_backup_replace.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/05500_object_storage_identity_cache_backup_replace.sh b/tests/queries/0_stateless/05500_object_storage_identity_cache_backup_replace.sh new file mode 100755 index 0000000000000..3af56a327f621 --- /dev/null +++ b/tests/queries/0_stateless/05500_object_storage_identity_cache_backup_replace.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-parallel-replicas +# Tag no-fasttest: reads a file through the mock S3 server at localhost:11111 +# Tag no-parallel: the identity cache is process-global, and another test may clear it +# Tag no-parallel-replicas: per-query S3 request counts change when a read is spread across replicas + +# A backup overwrites its objects through BackupWriterS3, not S3ObjectStorage. After the first backup is +# read, the identity cache holds that ETag. A second backup to the same URL must drop the entry, or the +# next read sends If-Match for the previous ETag and fails with S3_OBJECT_CHANGED_DURING_READ. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +BACKUP_URL="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}/" +OBJECT_URL="${BACKUP_URL}.backup" +COMMON_SETTINGS="max_threads = 1, use_hive_partitioning = 0, parallel_replicas_for_cluster_engines = 0, enable_filesystem_cache = 0, use_page_cache_for_object_storage = 0, use_object_storage_list_objects_cache = 0, s3_validate_etag_on_read = 1" + +fail() +{ + echo "FAIL: $*" >&2 + exit 1 +} + +# $1 label, $2 head mode (eq0 or ge1) +read_backup_and_check() +{ + local label=$1 + local head_mode=$2 + local qid="idcache_backup_${label}_${CLICKHOUSE_DATABASE}" + local data row head + + data=$(${CLICKHOUSE_CLIENT} --query_id "$qid" -q \ + "SELECT count() FROM s3('${OBJECT_URL}', 'test', 'testtest', 'LineAsString', 's String') SETTINGS ${COMMON_SETTINGS} FORMAT TabSeparated") \ + || fail "${label}: select failed" + [[ $data -ge 1 ]] || fail "${label}: data '${data}', expected at least 1 row" + + ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "${label}: flush logs failed" + row=$(${CLICKHOUSE_CLIENT} -q " + SELECT ProfileEvents['S3HeadObject'] + FROM system.query_log + WHERE event_date >= yesterday() + AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND query_id = '${qid}' + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC + LIMIT 1 + FORMAT TabSeparated") || fail "${label}: query_log failed" + [[ -n $row ]] || fail "${label}: no query_log row" + head=$row + case "$head_mode" in + eq0) [[ $head == "0" ]] || fail "${label}: S3HeadObject=${head}, expected 0" ;; + ge1) [[ $head -ge 1 ]] || fail "${label}: S3HeadObject=${head}, expected >= 1" ;; + *) fail "${label}: bad mode ${head_mode}" ;; + esac +} + +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear failed" +${CLICKHOUSE_CLIENT} -q "CREATE TABLE t (n UInt64) ENGINE = Memory" || fail "create failed" +${CLICKHOUSE_CLIENT} -q "INSERT INTO t VALUES (1)" || fail "insert failed" +${CLICKHOUSE_CLIENT} -q "BACKUP TABLE t TO S3('${BACKUP_URL}', 'test', 'testtest')" >/dev/null || fail "first backup failed" + +read_backup_and_check cold ge1 +read_backup_and_check warm eq0 + +${CLICKHOUSE_CLIENT} -q "INSERT INTO t VALUES (2)" || fail "second insert failed" +${CLICKHOUSE_CLIENT} -q "BACKUP TABLE t TO S3('${BACKUP_URL}', 'test', 'testtest')" >/dev/null || fail "second backup failed" + +read_backup_and_check replaced ge1 + +echo OK From 2d5058832c345577eb3b51314b1f24be5f0f1a4b Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Thu, 19 Nov 2026 13:47:49 +0100 Subject: [PATCH 33/37] Object storage: drop the identity cache entry when an upload fails after the object was written `WriteBufferFromS3::finalizeImpl` ran `on_upload_complete` only after `task_tracker->waitAll()` and the post-upload size check. An exception from an upload task skipped it. `PutObject` or `CompleteMultipartUpload` can commit the object and still report an error, for example when the response is lost, so the cached identity kept the previous ETag. The next read with `s3_validate_etag_on_read = 1` then failed with `S3_OBJECT_CHANGED_DURING_READ` once. The upload worker now sets `object_creation_request_sent` immediately before each object-creating request. `finalizeImpl` runs `waitAll` and the size check inside one `try`, and its `catch` calls `on_upload_complete` when that flag is set, then rethrows. A part upload that fails before any object-creating request leaves the flag unset, so the entry for the still-current object is kept. `ExceptionOnCompleteMPURunsUploadCallback` checks that a failed `CompleteMultipartUpload` runs the callback once and still throws. PR: https://github.com/Altinity/ClickHouse/pull/2888 Co-authored-by: Cursor --- src/IO/WriteBufferFromS3.cpp | 20 +++++++++----- src/IO/WriteBufferFromS3.h | 3 +++ src/IO/tests/gtest_writebuffer_s3.cpp | 38 +++++++++++++++++++++++++-- 3 files changed, 53 insertions(+), 8 deletions(-) diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 74f907f0fcae8..93fa2372867b1 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -227,12 +227,12 @@ void WriteBufferFromS3::finalizeImpl() chassert(offset() == 0); chassert(hidden_size == 0); - task_tracker->waitAll(); - - span.addAttributeIfNotZero("clickhouse.multipart_upload_parts", multipart_tags.size()); - try { + task_tracker->waitAll(); + + span.addAttributeIfNotZero("clickhouse.multipart_upload_parts", multipart_tags.size()); + if (request_settings[S3RequestSetting::check_objects_after_upload]) { S3::checkObjectExists(*client_ptr, bucket, key, {}, "Immediately after upload"); @@ -247,8 +247,11 @@ void WriteBufferFromS3::finalizeImpl() } catch (...) { - /// The upload already replaced the object. Drop any identity cached for the previous one. - if (on_upload_complete) + /// `PutObject` or `CompleteMultipartUpload` may have committed the object even when this + /// request failed (a lost response is reported as an error). Drop the cached identity so the + /// next read does not send the previous ETag. A part upload that failed before either request + /// leaves `object_creation_request_sent` false: the visible object is unchanged. + if (object_creation_request_sent && on_upload_complete) on_upload_complete(); throw; } @@ -720,6 +723,9 @@ bool WriteBufferFromS3::completeMultipartUpload() ProfileEvents::increment(ProfileEvents::DiskS3CompleteMultipartUpload); Stopwatch watch; + /// Set before the call. A failure after S3 has committed the upload must still be visible to + /// `finalizeImpl`, which drops the cached identity only once this request has been sent. + object_creation_request_sent = true; auto outcome = object_creation_client->CompleteMultipartUpload(req); auto elapsed = watch.elapsedMicroseconds(); @@ -826,6 +832,8 @@ void WriteBufferFromS3::makeSinglepartUpload(WriteBufferFromS3::PartData && data CurrentThread::WriteThrottlingScope write_throttling_scope(write_settings.remote_throttler); Stopwatch watch; + /// Set before the call. A lost `PutObject` response can still have replaced the object. + object_creation_request_sent = true; auto outcome = object_creation_client->PutObject(request); auto elapsed = watch.elapsedMicroseconds(); diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h index f7c0739bbc366..30a551b9db6bc 100644 --- a/src/IO/WriteBufferFromS3.h +++ b/src/IO/WriteBufferFromS3.h @@ -107,6 +107,9 @@ class WriteBufferFromS3 final : public WriteBufferFromFileBase /// finalize barrier (happens-before), so no extra synchronization is needed. std::optional object_etag; OnUploadComplete on_upload_complete; + /// Set by the upload worker immediately before `PutObject` or `CompleteMultipartUpload`. A lost + /// response can still have committed the object. Read after `waitAll` (happens-before). + bool object_creation_request_sent = false; LoggerPtr log = getLogger("WriteBufferFromS3"); LogSeriesLimiterPtr limited_log = std::make_shared(log, 1, 5); diff --git a/src/IO/tests/gtest_writebuffer_s3.cpp b/src/IO/tests/gtest_writebuffer_s3.cpp index 686a8b4d61626..d5465b71a301a 100644 --- a/src/IO/tests/gtest_writebuffer_s3.cpp +++ b/src/IO/tests/gtest_writebuffer_s3.cpp @@ -811,7 +811,10 @@ class WBS3Test : public ::testing::Test return *async_policy; } - std::unique_ptr getWriteBuffer(String file_name = "file", const WriteSettings & write_settings = {}) + std::unique_ptr getWriteBuffer( + String file_name = "file", + const WriteSettings & write_settings = {}, + WriteBufferFromS3::OnUploadComplete on_upload_complete = {}) { S3::S3RequestSettings request_settings; request_settings.updateFromSettings(settings, /* if_changed */true, /* validate_settings */false); @@ -829,7 +832,9 @@ class WBS3Test : public ::testing::Test nullptr, std::nullopt, getAsyncPolicy().getScheduler(), - write_settings); + write_settings, + nullptr, + std::move(on_upload_complete)); } void setInjectionModel(std::shared_ptr injections_) @@ -1067,6 +1072,35 @@ TEST_P(SyncAsync, ExceptionOnCreateMPU) { } +/// `CompleteMultipartUpload` can commit the object and still report an error (a lost response). +/// `finalize` must run the upload callback before propagating that error, so the identity cache +/// does not keep the previous ETag. A part-upload failure never sends the completing request and +/// must not run the callback. +TEST_P(SyncAsync, ExceptionOnCompleteMPURunsUploadCallback) { + setInjectionModel(std::make_shared()); + + getSettings()[Setting::s3_max_single_part_upload_size] = 0; // no single part + getSettings()[Setting::s3_min_upload_part_size] = 1; // small parts are ok + + size_t callbacks = 0; + EXPECT_THROW({ + try { + auto buffer = getWriteBuffer("exception_on_complete_mpu_callback", {}, [&]() { ++callbacks; }); + buffer->write('A'); + + getAsyncPolicy().setAutoExecute(true); + buffer->finalize(); + } + catch(const DB::Exception & e) + { + ASSERT_EQ(ErrorCodes::S3_ERROR, e.code()); + EXPECT_THAT(e.what(), testing::HasSubstr("CompleteMPUFailIngection")); + throw; + } + }, DB::S3Exception); + EXPECT_EQ(callbacks, 1u); +} + TEST_P(SyncAsync, ExceptionOnCompleteMPU) { setInjectionModel(std::make_shared()); From e1b15809e0625bb9c9038a52daca3ca518b6e775 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Thu, 19 Nov 2026 14:08:26 +0100 Subject: [PATCH 34/37] Object storage: drop the identity cache entry when a copy fails after the object was created `copyS3File`, `copyS3FileRange`, and `copyDataToS3File` invalidate the destination only when they return. `BackupWriterS3` and `S3ObjectStorage::copyObject` / `copyObjectToAnotherObjectStorage` relied on that. If `CompleteMultipartUpload`, `PutObject`, or `CopyObject` committed the destination and the response was lost, the call threw before the invalidation, so the cached identity kept the previous ETag. The next read with `s3_validate_etag_on_read = 1` sent it in `If-Match` and failed with `S3_OBJECT_CHANGED_DURING_READ` once. The copy helpers now take an `OnObjectCreationFailure` callback. `UploadHelper` sets `object_creation_request_sent` immediately before each object-creating request, and `performCopy` calls the callback from its `catch` only when that flag is set. A failure before any such request, for example a part upload that fails first, leaves the entry for the still-current object in place. `BackupWriterS3` passes the same invalidation it runs after a successful copy, and the disk copy paths in `S3ObjectStorage` do the same. The success path is unchanged. Tests `CopyDataToS3FileRunsCallbackWhenCompleteFails` and `CopyDataToS3FileSkipsCallbackWhenPartUploadFails` check that a failed `CompleteMultipartUpload` runs the callback once and a failed `UploadPart` does not. PR: https://github.com/Altinity/ClickHouse/pull/2888 Co-authored-by: Cursor --- src/Backups/BackupIO_S3.cpp | 36 +++++- .../ObjectStorages/S3/S3ObjectStorage.cpp | 12 +- src/IO/S3/copyS3File.cpp | 106 ++++++++++++++---- src/IO/S3/copyS3File.h | 15 ++- src/IO/tests/gtest_writebuffer_s3.cpp | 91 +++++++++++++++ 5 files changed, 231 insertions(+), 29 deletions(-) diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp index bfe936871fabd..b426e94ecaeb4 100644 --- a/src/Backups/BackupIO_S3.cpp +++ b/src/Backups/BackupIO_S3.cpp @@ -671,6 +671,14 @@ void BackupWriterS3::copyFileFromDisk( : (copy_encrypted ? src_disk->getEncryptedFileSize(src_path) : src_disk->getFileSize(src_path)); const bool whole_object = (src_offset == 0) && (length == source_size); const String dest_key = fs::path(s3_uri.key) / path_in_backup; + /// Runs when the copy throws after `CopyObject`, `PutObject`, or `CompleteMultipartUpload` + /// was sent. The success path below invalidates only if the copy returns. + const String endpoint = s3_uri.endpoint; + const String bucket = s3_uri.bucket; + auto on_object_creation_failure = [endpoint, bucket, dest_key]() + { + invalidateObjectStorageIdentityCache(endpoint, bucket, dest_key); + }; if (whole_object) { @@ -687,7 +695,11 @@ void BackupWriterS3::copyFileFromDisk( read_settings, blob_storage_log, runner, - create_read_buffer); + create_read_buffer, + /* object_metadata= */ std::nullopt, + ObjectStorageCopyMode::Default, + /* object_creation_client= */ nullptr, + on_object_creation_failure); } else { @@ -705,7 +717,9 @@ void BackupWriterS3::copyFileFromDisk( read_settings, blob_storage_log, runner, - create_read_buffer); + create_read_buffer, + /* object_metadata= */ std::nullopt, + on_object_creation_failure); } /// The copy has replaced `dest_key`. A read that cached the previous ETag must not send it as `If-Match`. @@ -724,6 +738,8 @@ void BackupWriterS3::copyFile(const String & destination, const String & source, const auto source_key = fs::path(s3_uri.key) / source; const String dest_key = fs::path(s3_uri.key) / destination; + const String endpoint = s3_uri.endpoint; + const String bucket = s3_uri.bucket; copyS3File( client, /* src_bucket */ s3_uri.bucket, @@ -741,6 +757,13 @@ void BackupWriterS3::copyFile(const String & destination, const String & source, LOG_TRACE(log, "Falling back to copy file inside backup from {} to {} through direct buffers", source, destination); return std::make_unique( client, s3_uri.bucket, source_key, s3_uri.version_id, s3_settings.request_settings, read_settings); + }, + /* object_metadata= */ std::nullopt, + ObjectStorageCopyMode::Default, + /* object_creation_client= */ nullptr, + [endpoint, bucket, dest_key]() + { + invalidateObjectStorageIdentityCache(endpoint, bucket, dest_key); }); invalidateObjectStorageIdentityCache(s3_uri.endpoint, s3_uri.bucket, dest_key); } @@ -748,9 +771,16 @@ void BackupWriterS3::copyFile(const String & destination, const String & source, void BackupWriterS3::copyDataToFile(const String & path_in_backup, const CreateReadBufferFunction & create_read_buffer, UInt64 start_pos, UInt64 length) { const String dest_key = fs::path(s3_uri.key) / path_in_backup; + const String endpoint = s3_uri.endpoint; + const String bucket = s3_uri.bucket; copyDataToS3File(create_read_buffer, start_pos, length, client, s3_uri.bucket, dest_key, s3_settings.request_settings, blob_storage_log, - threadPoolCallbackRunnerUnsafe(getBackupsIOThreadPool().get(), ThreadName::S3_BACKUP_WRITER)); + threadPoolCallbackRunnerUnsafe(getBackupsIOThreadPool().get(), ThreadName::S3_BACKUP_WRITER), + /* object_metadata= */ std::nullopt, + [endpoint, bucket, dest_key]() + { + invalidateObjectStorageIdentityCache(endpoint, bucket, dest_key); + }); invalidateObjectStorageIdentityCache(s3_uri.endpoint, s3_uri.bucket, dest_key); } diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp index adf2fe46e1c2c..86fad7052e780 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp @@ -1223,7 +1223,11 @@ void S3ObjectStorage::copyObjectToAnotherObjectStorage( // NOLINT [&, this]{ return readObject(object_from, read_settings_to_use);}, object_to_attributes, write_settings.object_storage_copy_mode, - dest_s3->objectCreationClient(write_settings)); + dest_s3->objectCreationClient(write_settings), + [dest_s3, dest_path = object_to.remote_path]() + { + dest_s3->invalidateCachedIdentity(dest_path); + }); dest_s3->invalidateCachedIdentity(object_to.remote_path); return; } @@ -1303,7 +1307,11 @@ void S3ObjectStorage::copyObject( // NOLINT [&, this]{ return readObject(object_from, read_settings_to_use);}, object_to_attributes, write_settings.object_storage_copy_mode, - objectCreationClient(write_settings)); + objectCreationClient(write_settings), + [this, dest_path = object_to.remote_path]() + { + invalidateCachedIdentity(dest_path); + }); invalidateCachedIdentity(object_to.remote_path); } diff --git a/src/IO/S3/copyS3File.cpp b/src/IO/S3/copyS3File.cpp index 76f2dbb2df575..10db4a450f146 100644 --- a/src/IO/S3/copyS3File.cpp +++ b/src/IO/S3/copyS3File.cpp @@ -92,7 +92,8 @@ namespace ThreadPoolCallbackRunnerUnsafe schedule_, BlobStorageLogWriterPtr blob_storage_log_, const LoggerPtr log_, - const std::shared_ptr & object_creation_client_ = nullptr) + const std::shared_ptr & object_creation_client_ = nullptr, + OnObjectCreationFailure on_object_creation_failure_ = {}) : client_ptr(client_ptr_) , object_creation_client(object_creation_client_ ? object_creation_client_ : client_ptr_) , dest_bucket(dest_bucket_) @@ -102,6 +103,7 @@ namespace , schedule(schedule_) , blob_storage_log(blob_storage_log_) , log(log_) + , on_object_creation_failure(std::move(on_object_creation_failure_)) , num_parts(0) , normal_part_size(0) { @@ -120,6 +122,11 @@ namespace ThreadPoolCallbackRunnerUnsafe schedule; BlobStorageLogWriterPtr blob_storage_log; const LoggerPtr log; + /// Called from `performCopy` when this copy throws after an object-creating request was sent. + OnObjectCreationFailure on_object_creation_failure; + /// Set immediately before `CopyObject`, `PutObject`, or `CompleteMultipartUpload`. A lost + /// response can still have committed the destination. Read on the same thread that writes it. + bool object_creation_request_sent = false; /// Represents a task uploading a single part. /// Keep this struct small because there can be thousands of parts. @@ -139,6 +146,12 @@ namespace std::atomic num_finished_parts = 0; std::atomic has_failed = false; + void notifyObjectCreationFailure() + { + if (object_creation_request_sent && on_object_creation_failure) + on_object_creation_failure(); + } + void fillCreateMultipartRequest(S3::CreateMultipartUploadRequest & request) { request.SetBucket(dest_bucket); @@ -219,6 +232,9 @@ namespace ProfileEvents::increment(ProfileEvents::DiskS3CompleteMultipartUpload); Stopwatch watch; + /// Set before the call. A failure after S3 has committed the upload must still reach + /// `performCopy`, which drops the cached identity only once this request has been sent. + object_creation_request_sent = true; auto outcome = object_creation_client->CompleteMultipartUpload(request); auto elapsed = watch.elapsedMicroseconds(); @@ -450,8 +466,19 @@ namespace const S3::S3RequestSettings & request_settings_, const std::optional & object_metadata_, ThreadPoolCallbackRunnerUnsafe schedule_, - BlobStorageLogWriterPtr blob_storage_log_) - : UploadHelper(client_ptr_, dest_bucket_, dest_key_, request_settings_, object_metadata_, schedule_, blob_storage_log_, getLogger("copyDataToS3File")) + BlobStorageLogWriterPtr blob_storage_log_, + OnObjectCreationFailure on_object_creation_failure_) + : UploadHelper( + client_ptr_, + dest_bucket_, + dest_key_, + request_settings_, + object_metadata_, + schedule_, + blob_storage_log_, + getLogger("copyDataToS3File"), + /* object_creation_client_= */ nullptr, + std::move(on_object_creation_failure_)) , create_read_buffer(create_read_buffer_) , offset(offset_) , size(size_) @@ -460,13 +487,21 @@ namespace void performCopy() { - if (size <= request_settings[S3RequestSetting::max_single_part_upload_size]) - performSinglepartUpload(); - else - performMultipartUpload(); + try + { + if (size <= request_settings[S3RequestSetting::max_single_part_upload_size]) + performSinglepartUpload(); + else + performMultipartUpload(); - if (request_settings[S3RequestSetting::check_objects_after_upload]) - checkObjectAfterUpload(); + if (request_settings[S3RequestSetting::check_objects_after_upload]) + checkObjectAfterUpload(); + } + catch (...) + { + notifyObjectCreationFailure(); + throw; + } } private: @@ -521,6 +556,8 @@ namespace ProfileEvents::increment(ProfileEvents::DiskS3PutObject); Stopwatch watch; + /// Set before the call. A lost `PutObject` response can still have committed the object. + object_creation_request_sent = true; auto outcome = client_ptr->PutObject(request); auto elapsed = watch.elapsedMicroseconds(); @@ -647,7 +684,8 @@ namespace std::function fallback_method_, bool is_ranged_copy_, bool allow_fallback_, - const std::shared_ptr & object_creation_client_) + const std::shared_ptr & object_creation_client_, + OnObjectCreationFailure on_object_creation_failure_) : UploadHelper( client_ptr_, dest_bucket_, @@ -657,7 +695,8 @@ namespace schedule_, blob_storage_log_, getLogger("copyS3File"), - object_creation_client_) + object_creation_client_, + std::move(on_object_creation_failure_)) , src_bucket(src_bucket_) , src_key(src_key_) , offset(src_offset_) @@ -672,6 +711,19 @@ namespace } void performCopy() + { + try + { + performCopyImpl(); + } + catch (...) + { + notifyObjectCreationFailure(); + throw; + } + } + + void performCopyImpl() { LOG_TEST(log, "Copy object {} to {} using native copy", src_key, dest_key); @@ -759,6 +811,8 @@ namespace if (client_ptr->isClientForDisk()) ProfileEvents::increment(ProfileEvents::DiskS3CopyObject); + /// Set before the call. A lost `CopyObject` response can still have replaced the destination. + object_creation_request_sent = true; auto outcome = object_creation_client->CopyObject(request); if (outcome.IsSuccess()) { @@ -915,7 +969,8 @@ void copyDataToS3File( const S3::S3RequestSettings & settings, BlobStorageLogWriterPtr blob_storage_log, ThreadPoolCallbackRunnerUnsafe schedule, - const std::optional & object_metadata) + const std::optional & object_metadata, + OnObjectCreationFailure on_object_creation_failure) { CopyDataToFileHelper helper{ create_read_buffer, @@ -927,7 +982,8 @@ void copyDataToS3File( settings, object_metadata, schedule, - blob_storage_log}; + blob_storage_log, + std::move(on_object_creation_failure)}; helper.performCopy(); } @@ -954,12 +1010,14 @@ namespace const std::optional & object_metadata, bool is_ranged_copy, ObjectStorageCopyMode copy_mode, - std::shared_ptr object_creation_client) + std::shared_ptr object_creation_client, + OnObjectCreationFailure on_object_creation_failure) { if (!dest_s3_client) dest_s3_client = src_s3_client; - std::function fallback_method = [&] mutable + /// Copy the callback. The native helper takes the original, and this fallback can run after that move. + std::function fallback_method = [&, on_object_creation_failure]() mutable { copyDataToS3File( fallback_file_reader, @@ -971,7 +1029,8 @@ namespace settings, blob_storage_log, schedule, - object_metadata); + object_metadata, + on_object_creation_failure); }; if (!settings[S3RequestSetting::allow_native_copy]) @@ -1003,7 +1062,8 @@ namespace std::move(fallback_method), is_ranged_copy, /*allow_fallback=*/copy_mode == ObjectStorageCopyMode::Default, - object_creation_client}; + object_creation_client, + std::move(on_object_creation_failure)}; helper.performCopy(); } } @@ -1023,7 +1083,8 @@ void copyS3File( const CreateReadBuffer & fallback_file_reader, const std::optional & object_metadata, ObjectStorageCopyMode copy_mode, - std::shared_ptr object_creation_client) + std::shared_ptr object_creation_client, + OnObjectCreationFailure on_object_creation_failure) { /// The read-write fallback would write through the retrying client. chassert(!object_creation_client || copy_mode == ObjectStorageCopyMode::NativeOnly); @@ -1046,7 +1107,8 @@ void copyS3File( object_metadata, /* is_ranged_copy= */ false, copy_mode, - std::move(object_creation_client)); + std::move(object_creation_client), + std::move(on_object_creation_failure)); } void copyS3FileRange( @@ -1064,7 +1126,8 @@ void copyS3FileRange( BlobStorageLogWriterPtr blob_storage_log, ThreadPoolCallbackRunnerUnsafe schedule, const CreateReadBuffer & fallback_file_reader, - const std::optional & object_metadata) + const std::optional & object_metadata, + OnObjectCreationFailure on_object_creation_failure) { copyS3FileImpl( std::move(src_s3_client), @@ -1084,7 +1147,8 @@ void copyS3FileRange( object_metadata, /* is_ranged_copy= */ true, ObjectStorageCopyMode::Default, - /* object_creation_client= */ nullptr); + /* object_creation_client= */ nullptr, + std::move(on_object_creation_failure)); } } diff --git a/src/IO/S3/copyS3File.h b/src/IO/S3/copyS3File.h index 9106070471bab..1a3aef0e903fd 100644 --- a/src/IO/S3/copyS3File.h +++ b/src/IO/S3/copyS3File.h @@ -21,6 +21,12 @@ class StdStreamFromReadBuffer; using CreateReadBuffer = std::function()>; +/// Invoked when `copyS3File`, `copyS3FileRange`, or `copyDataToS3File` throws after `CopyObject`, +/// `PutObject`, or `CompleteMultipartUpload` was sent. That request may have committed the +/// destination. A failure before any of those requests does not call it. A successful copy does +/// not call it either: the caller drops a cached identity after the function returns. +using OnObjectCreationFailure = std::function; + /// Builds the S3 upload request body for the part [offset, offset + size) of the source. /// The part is read fully into memory up front, so the returned body has no failable inner /// source buffer: the AWS SDK can rewind (clear(); seekg(0)) and resend the body on a retry @@ -60,7 +66,8 @@ void copyS3File( const CreateReadBuffer & fallback_file_reader, const std::optional & object_metadata = std::nullopt, ObjectStorageCopyMode copy_mode = ObjectStorageCopyMode::Default, - std::shared_ptr object_creation_client = nullptr); + std::shared_ptr object_creation_client = nullptr, + OnObjectCreationFailure on_object_creation_failure = {}); /// Copies exactly `[src_offset, src_offset + src_size)` of a LARGER source object of size `src_object_size`. /// @@ -83,7 +90,8 @@ void copyS3FileRange( BlobStorageLogWriterPtr blob_storage_log, ThreadPoolCallbackRunnerUnsafe schedule, const CreateReadBuffer & fallback_file_reader, - const std::optional & object_metadata = std::nullopt); + const std::optional & object_metadata = std::nullopt, + OnObjectCreationFailure on_object_creation_failure = {}); /// Copies data from any seekable source to S3. /// The same functionality can be done by using the function copyData() and the class WriteBufferFromS3 @@ -100,7 +108,8 @@ void copyDataToS3File( const S3::S3RequestSettings & settings, BlobStorageLogWriterPtr blob_storage_log, ThreadPoolCallbackRunnerUnsafe schedule, - const std::optional & object_metadata = std::nullopt); + const std::optional & object_metadata = std::nullopt, + OnObjectCreationFailure on_object_creation_failure = {}); } diff --git a/src/IO/tests/gtest_writebuffer_s3.cpp b/src/IO/tests/gtest_writebuffer_s3.cpp index d5465b71a301a..da386b8e4ca31 100644 --- a/src/IO/tests/gtest_writebuffer_s3.cpp +++ b/src/IO/tests/gtest_writebuffer_s3.cpp @@ -1195,6 +1195,97 @@ TEST_F(WBS3Test, CopyDataToS3FileRetriesInvalidPart) { EXPECT_EQ(bStore.objects["copy_data_invalid_part_retry"].size(), payload.size()); } +/// `CompleteMultipartUpload` can commit the object and still report an error. `copyDataToS3File` must +/// run the failure callback before propagating that error, so a backup copy drops the cached identity. +/// An `UploadPart` failure never sends the completing request and must not run the callback. +TEST_F(WBS3Test, CopyDataToS3FileRunsCallbackWhenCompleteFails) +{ + setInjectionModel(std::make_shared()); + + getSettings()[Setting::s3_max_single_part_upload_size] = 0; + getSettings()[Setting::s3_min_upload_part_size] = 1; + getSettings()[Setting::s3_check_objects_after_upload] = false; + + S3::S3RequestSettings request_settings; + request_settings.updateFromSettings(settings, /* if_changed */ true, /* validate_settings */ false); + + const String payload = "copy_complete_fail_payload"; + auto create_read_buffer = [&]() -> std::unique_ptr + { + return std::make_unique(payload); + }; + + size_t callbacks = 0; + EXPECT_THROW({ + try + { + copyDataToS3File( + create_read_buffer, + /* offset= */ 0, + /* size= */ payload.size(), + client, + bucket, + "copy_data_complete_fail_callback", + request_settings, + /* blob_storage_log= */ nullptr, + /* schedule= */ {}, + /* object_metadata= */ std::nullopt, + [&]() { ++callbacks; }); + } + catch (const DB::Exception & e) + { + ASSERT_EQ(ErrorCodes::S3_ERROR, e.code()); + EXPECT_THAT(e.what(), testing::HasSubstr("CompleteMPUFailIngection")); + throw; + } + }, DB::S3Exception); + EXPECT_EQ(callbacks, 1u); +} + +TEST_F(WBS3Test, CopyDataToS3FileSkipsCallbackWhenPartUploadFails) +{ + setInjectionModel(std::make_shared()); + + getSettings()[Setting::s3_max_single_part_upload_size] = 0; + getSettings()[Setting::s3_min_upload_part_size] = 1; + getSettings()[Setting::s3_check_objects_after_upload] = false; + + S3::S3RequestSettings request_settings; + request_settings.updateFromSettings(settings, /* if_changed */ true, /* validate_settings */ false); + + const String payload = "copy_part_fail_payload"; + auto create_read_buffer = [&]() -> std::unique_ptr + { + return std::make_unique(payload); + }; + + size_t callbacks = 0; + EXPECT_THROW({ + try + { + copyDataToS3File( + create_read_buffer, + /* offset= */ 0, + /* size= */ payload.size(), + client, + bucket, + "copy_data_part_fail_callback", + request_settings, + /* blob_storage_log= */ nullptr, + /* schedule= */ {}, + /* object_metadata= */ std::nullopt, + [&]() { ++callbacks; }); + } + catch (const DB::Exception & e) + { + ASSERT_EQ(ErrorCodes::S3_ERROR, e.code()); + EXPECT_THAT(e.what(), testing::HasSubstr("UploadPartFailIngection")); + throw; + } + }, DB::S3Exception); + EXPECT_EQ(callbacks, 0u); +} + /// copyS3File routing between whole-object CopyObject and ranged UploadPartCopy. A small copy would take /// CopyObject, which carries no byte range and copies the ENTIRE source; a partial-range copy must therefore /// force UploadPartCopy, which sets a CopySourceRange per part -- but only when S3 would accept the source as From 1263d350b25c267f806aa5f44243b563555b00a7 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Thu, 19 Nov 2026 14:17:18 +0100 Subject: [PATCH 35/37] Object storage queue: drop the identity cache entry for a cross-bucket move `ObjectStorageQueuePostProcessor::moveS3Objects` copies each object into another bucket with `copyS3File` directly, not through `S3ObjectStorage::copyObject`. Nothing dropped the destination's entry in `ObjectStorageIdentityCache`, so a key read earlier through `s3()` or an S3 table kept its previous ETag. After the move overwrote it, the next read with `s3_validate_etag_on_read = 1` sent that ETag in `If-Match`, got a 412, and failed with `S3_OBJECT_CHANGED_DURING_READ` once. The move now passes an `on_object_creation_failure` callback to `copyS3File` and calls the same invalidation after the copy returns. The callback runs only when `CopyObject`, `PutObject`, or `CompleteMultipartUpload` was sent and the copy then threw. A failure before that leaves the entry for the still-current object in place. A move within the same bucket is unchanged, because it already goes through `S3ObjectStorage::copyObject`. Azure moves do not use this cache. PR: https://github.com/Altinity/ClickHouse/pull/2888 Co-authored-by: Cursor --- .../ObjectStorageQueuePostProcessor.cpp | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueuePostProcessor.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueuePostProcessor.cpp index c520d97da2bb0..0328a319b1e81 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueuePostProcessor.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueuePostProcessor.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -363,6 +364,25 @@ void ObjectStorageQueuePostProcessor::moveS3Objects(const StoredObjects & object object_from.remote_path); auto object_to = applyMovePrefixIfPresent(object_from, move_prefix, settings.after_processing_move_preserve_path); + /// This copy does not go through `S3ObjectStorage`, so nothing else drops the + /// destination's cached identity. A lost `CopyObject` or `CompleteMultipartUpload` + /// response can still have replaced the object, so the callback covers that throw. + const String endpoint = dst_uri.endpoint; + const String bucket = dst_uri.bucket; + const String dest_key = object_to.remote_path; + auto invalidate_destination = [endpoint, bucket, dest_key]() + { + auto global_context = Context::getGlobalContextInstance(); + if (!global_context) + return; + + auto cache = global_context->getObjectStorageIdentityCache(); + if (!cache) + return; + + cache->invalidate(makeObjectStorageIdentityCacheKey(endpoint, bucket, dest_key)); + }; + LOG_INFO(log, "Copying {} ({} Bytes) to bucket {}", object_from.remote_path, object_size, dst_uri.bucket); copyS3File( src_client, @@ -371,15 +391,20 @@ void ObjectStorageQueuePostProcessor::moveS3Objects(const StoredObjects & object /*src_size=*/ object_size, /*dest_s3_client=*/ dst_client, /*dest_bucket=*/ dst_uri.bucket, - /*dest_key=*/ object_to.remote_path, + /*dest_key=*/ dest_key, /*settings=*/ s3_settings->request_settings, /*read_settings=*/ read_settings_to_use, BlobStorageLogWriter::create(object_storage->getDiskName()), scheduler, - /*fallback_file_reader=*/ [&]{ + /*fallback_file_reader=*/ [&] + { return s3_storage->readObject(object_from, read_settings_to_use); - } - /*object_metadata=*/); + }, + /*object_metadata=*/ std::nullopt, + ObjectStorageCopyMode::Default, + /*object_creation_client=*/ nullptr, + invalidate_destination); + invalidate_destination(); LOG_INFO(log, "Removing object {}", object_from.remote_path); object_storage->removeObjectIfExists(object_from); From 2add0efaaadbc4262c601f90d3943ff6e31df887 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Thu, 19 Nov 2026 14:29:11 +0100 Subject: [PATCH 36/37] Object storage: share one helper for dropping the identity cache entry The backup writer, the queue move, and the S3 disk each looked up the global context and the identity cache and invalidated one key. That lookup is now `invalidateObjectStorageIdentityCache`, declared next to `makeObjectStorageIdentityCacheKey` in `ObjectStorageIdentityCache.h` and defined in `ObjectStorageIdentityCache.cpp`. It does nothing when the global context or the cache is not set. `BackupIO_S3.cpp` drops its local copy. `ObjectStorageQueuePostProcessor::moveS3Objects` calls the shared function from its invalidation lambda. `S3ObjectStorage::invalidateCachedIdentity` forwards to it with the disk's endpoint and bucket, and the read and write buffer callbacks call it with copies of the endpoint, bucket, and path, because those buffers can outlive the storage. No behavior change. PR: https://github.com/Altinity/ClickHouse/pull/2888 Co-authored-by: Cursor --- src/Backups/BackupIO_S3.cpp | 20 ----------- .../ObjectStorageIdentityCache.cpp | 14 ++++++++ .../ObjectStorageIdentityCache.h | 4 +++ .../ObjectStorages/S3/S3ObjectStorage.cpp | 33 ++++++++----------- .../ObjectStorageQueuePostProcessor.cpp | 10 +----- 5 files changed, 33 insertions(+), 48 deletions(-) diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp index b426e94ecaeb4..5d24cfa3bdacc 100644 --- a/src/Backups/BackupIO_S3.cpp +++ b/src/Backups/BackupIO_S3.cpp @@ -358,26 +358,6 @@ class S3BackupClientCreator return res; } - - /// Drop the identity cached for `key`. Backup IO writes and deletes through `WriteBufferFromS3`, - /// `copyS3File`, and `deleteFileFromS3`, not through `S3ObjectStorage`, so nothing else drops the entry. - /// The next read HEADs instead of sending `If-Match` for the previous ETag. - void invalidateObjectStorageIdentityCache(const String & endpoint, const String & bucket, const String & key) - { - auto global_context = Context::getGlobalContextInstance(); - if (!global_context) - { - return; - } - - auto cache = global_context->getObjectStorageIdentityCache(); - if (!cache) - { - return; - } - - cache->invalidate(makeObjectStorageIdentityCacheKey(endpoint, bucket, key)); - } } diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp index a51f1f73be0dc..0b7f6988281e9 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp @@ -2,6 +2,7 @@ #include #include +#include namespace ProfileEvents { @@ -18,6 +19,19 @@ namespace CurrentMetrics namespace DB { +void invalidateObjectStorageIdentityCache(std::string_view endpoint, std::string_view bucket, std::string_view path) +{ + auto global_context = Context::getGlobalContextInstance(); + if (!global_context) + return; + + auto cache = global_context->getObjectStorageIdentityCache(); + if (!cache) + return; + + cache->invalidate(makeObjectStorageIdentityCacheKey(endpoint, bucket, path)); +} + ObjectStorageIdentityCache::ObjectStorageIdentityCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio) : Base( cache_policy, diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h index f9b4ae3b2a1f3..c57eb2ad29fff 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h @@ -66,6 +66,10 @@ inline String makeObjectStorageIdentityCacheKey(std::string_view endpoint, std:: return key; } +/// Drop the identity cached for this endpoint, bucket, and path. No-op when the global context or the +/// cache is not set. The next read of the path requests the metadata again. +void invalidateObjectStorageIdentityCache(std::string_view endpoint, std::string_view bucket, std::string_view path); + /// Cache of object-storage identity, keyed by endpoint, bucket and object path, so opening an object /// does not require a fresh HEAD on every read (a lake scan otherwise re-HEADs /// every candidate file on every query). Backed by CacheBase (LRU/SLRU) for size-bounded eviction diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp index 86fad7052e780..8c12bb0109374 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp @@ -401,9 +401,12 @@ std::unique_ptr S3ObjectStorage::readObject( /// NOLINT } } - /// The buffer can outlive this storage. A GET that finds the object replaced or missing drops the - /// cached identity, so the next read requests the metadata again instead of reusing it. - const String identity_key = makeObjectStorageIdentityCacheKey(uri.endpoint, uri.bucket, object.remote_path); + /// The buffer can outlive this storage, so the callback keeps its own copy of the key parts. + /// A GET that finds the object replaced or missing drops the cached identity, and the next read + /// requests the metadata again instead of reusing it. + const String endpoint = uri.endpoint; + const String bucket = uri.bucket; + const String path = object.remote_path; return std::make_unique( clientForRetryProfile(ObjectStorageControlRequest{ @@ -427,11 +430,9 @@ std::unique_ptr S3ObjectStorage::readObject( /// NOLINT refresh_callback, std::move(blob_storage_log), object.etag, - [identity_key]() + [endpoint, bucket, path]() { - if (auto global_context = Context::getGlobalContextInstance()) - if (auto cache = global_context->getObjectStorageIdentityCache()) - cache->invalidate(identity_key); + invalidateObjectStorageIdentityCache(endpoint, bucket, path); }); } @@ -513,7 +514,9 @@ std::unique_ptr S3ObjectStorage::writeObject( /// NOLIN .connect_timeout_cap_ms = write_settings.object_storage_connect_timeout_cap_ms}); /// Captured by value. The buffer can outlive this storage, and the cache itself is process-global. - const String identity_key = makeObjectStorageIdentityCacheKey(uri.endpoint, uri.bucket, object.remote_path); + const String endpoint = uri.endpoint; + const String bucket = uri.bucket; + const String path = object.remote_path; return std::make_unique( used_client, @@ -526,11 +529,9 @@ std::unique_ptr S3ObjectStorage::writeObject( /// NOLIN std::move(scheduler), disk_write_settings, objectCreationClient(write_settings), - [identity_key]() + [endpoint, bucket, path]() { - if (auto global_context = Context::getGlobalContextInstance()) - if (auto cache = global_context->getObjectStorageIdentityCache()) - cache->invalidate(identity_key); + invalidateObjectStorageIdentityCache(endpoint, bucket, path); }); } @@ -1527,13 +1528,7 @@ bool S3ObjectStorage::hasSingleAttemptClientForTest(uint64_t request_timeout_ms, void S3ObjectStorage::invalidateCachedIdentity(const std::string & path) const { - auto global_context = Context::getGlobalContextInstance(); - if (!global_context) - return; - auto cache = global_context->getObjectStorageIdentityCache(); - if (!cache) - return; - cache->invalidate(makeObjectStorageIdentityCacheKey(uri.endpoint, uri.bucket, path)); + invalidateObjectStorageIdentityCache(uri.endpoint, uri.bucket, path); } std::shared_ptr S3ObjectStorage::clientForRetryProfile(const ObjectStorageControlRequest & request) const diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueuePostProcessor.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueuePostProcessor.cpp index 0328a319b1e81..4bb5ba146c246 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueuePostProcessor.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueuePostProcessor.cpp @@ -372,15 +372,7 @@ void ObjectStorageQueuePostProcessor::moveS3Objects(const StoredObjects & object const String dest_key = object_to.remote_path; auto invalidate_destination = [endpoint, bucket, dest_key]() { - auto global_context = Context::getGlobalContextInstance(); - if (!global_context) - return; - - auto cache = global_context->getObjectStorageIdentityCache(); - if (!cache) - return; - - cache->invalidate(makeObjectStorageIdentityCacheKey(endpoint, bucket, dest_key)); + invalidateObjectStorageIdentityCache(endpoint, bucket, dest_key); }; LOG_INFO(log, "Copying {} ({} Bytes) to bucket {}", object_from.remote_path, object_size, dst_uri.bucket); From 9990639dacba12b56cfdce0a8180d33b3244ed18 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Thu, 19 Nov 2026 17:00:52 +0100 Subject: [PATCH 37/37] Object storage: declare s3_validate_etag_on_read in S3ObjectStorage `S3ObjectStorage::getObjectMetadataImpl` reads `Setting::s3_validate_etag_on_read` to decide whether to consult the identity cache, but the file never declared that setting in its `namespace Setting` block. The build failed with "no member named 's3_validate_etag_on_read' in namespace 'DB::Setting'". Add the same `extern` declaration the other object-storage readers already use. The setting itself is defined in `Settings.cpp`. PR: https://github.com/Altinity/ClickHouse/pull/2888 Co-authored-by: Cursor --- .../DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp index 8c12bb0109374..359576e0b5ff6 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp @@ -90,6 +90,7 @@ static UInt64 s3CredentialHash(const S3::Client & client) namespace Setting { extern const SettingsBool s3_validate_request_settings; + extern const SettingsBool s3_validate_etag_on_read; } namespace S3RequestSetting