diff --git a/docs/reference/statements/system.mdx b/docs/reference/statements/system.mdx index 24cd5a61ae96f..53f7c08045e37 100644 --- a/docs/reference/statements/system.mdx +++ b/docs/reference/statements/system.mdx @@ -134,6 +134,10 @@ Clears the per-URL Confluent Schema Registry caches used by the `AvroConfluent` Clears the parquet metadata cache. +## SYSTEM CLEAR|DROP OBJECT STORAGE IDENTITY CACHE {#drop-object-storage-identity-cache} + +Clears the object-storage identity cache, which stores per-object size, ETag and Last-Modified so that opening an object does not repeat a `HEAD` request. Its size is configured with the server settings `object_storage_identity_cache_size`, `object_storage_identity_cache_policy` and `object_storage_identity_cache_size_ratio`. Set `object_storage_identity_cache_size` to `0` to disable the cache. `SYSTEM RELOAD CONFIG` with a non-zero size creates it when startup left it disabled. A write, copy, or delete by this server drops that object's entry, and the next read requests the metadata again. + ## SYSTEM CLEAR|DROP PAIMON METADATA CACHE {#drop-paimon-metadata-cache} Clears the in-memory cache of parsed Paimon metadata files (manifest lists and manifests). diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index e8d02648c755f..b1966840602ae 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -169,6 +169,9 @@ namespace ServerSetting extern const ServerSettingsString mark_cache_policy; extern const ServerSettingsUInt64 mark_cache_size; extern const ServerSettingsDouble mark_cache_size_ratio; + extern const ServerSettingsString object_storage_identity_cache_policy; + extern const ServerSettingsUInt64 object_storage_identity_cache_size; + extern const ServerSettingsDouble object_storage_identity_cache_size_ratio; extern const ServerSettingsString unique_key_bitmap_cache_policy; extern const ServerSettingsUInt64 unique_key_bitmap_cache_size_bytes; extern const ServerSettingsDouble unique_key_bitmap_cache_size_ratio; @@ -1529,6 +1532,17 @@ void LocalServer::processConfig() } global_context->setMarkCache(mark_cache_policy, mark_cache_size, mark_cache_size_ratio); + String object_storage_identity_cache_policy = server_settings[ServerSetting::object_storage_identity_cache_policy]; + size_t object_storage_identity_cache_size = server_settings[ServerSetting::object_storage_identity_cache_size]; + double object_storage_identity_cache_size_ratio = server_settings[ServerSetting::object_storage_identity_cache_size_ratio]; + if (object_storage_identity_cache_size > max_cache_size) + { + object_storage_identity_cache_size = max_cache_size; + LOG_INFO(log, "Lowered object storage identity cache size to {} because the system has limited RAM", formatReadableSizeWithBinarySuffix(object_storage_identity_cache_size)); + } + global_context->setObjectStorageIdentityCache( + object_storage_identity_cache_policy, object_storage_identity_cache_size, object_storage_identity_cache_size_ratio); + /// UNIQUE KEY delete-bitmap cache. Zero size disables. String unique_key_bitmap_cache_policy_name = server_settings[ServerSetting::unique_key_bitmap_cache_policy]; size_t unique_key_bitmap_cache_size = server_settings[ServerSetting::unique_key_bitmap_cache_size_bytes]; diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 45947fb9b9cc6..8af11d18091f1 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -320,6 +320,9 @@ namespace ServerSetting extern const ServerSettingsString mark_cache_policy; extern const ServerSettingsUInt64 mark_cache_size; extern const ServerSettingsDouble mark_cache_size_ratio; + extern const ServerSettingsString object_storage_identity_cache_policy; + extern const ServerSettingsUInt64 object_storage_identity_cache_size; + extern const ServerSettingsDouble object_storage_identity_cache_size_ratio; extern const ServerSettingsString unique_key_index_cache_policy; extern const ServerSettingsUInt64 unique_key_index_cache_size_bytes; extern const ServerSettingsDouble unique_key_index_cache_size_ratio; @@ -2475,6 +2478,17 @@ try } global_context->setMarkCache(mark_cache_policy, mark_cache_size, mark_cache_size_ratio); + String object_storage_identity_cache_policy = server_settings[ServerSetting::object_storage_identity_cache_policy]; + size_t object_storage_identity_cache_size = server_settings[ServerSetting::object_storage_identity_cache_size]; + double object_storage_identity_cache_size_ratio = server_settings[ServerSetting::object_storage_identity_cache_size_ratio]; + if (object_storage_identity_cache_size > max_cache_size) + { + object_storage_identity_cache_size = max_cache_size; + LOG_INFO(log, "Lowered object storage identity cache size to {} because the system has limited RAM", formatReadableSizeWithBinarySuffix(object_storage_identity_cache_size)); + } + global_context->setObjectStorageIdentityCache( + object_storage_identity_cache_policy, object_storage_identity_cache_size, object_storage_identity_cache_size_ratio); + String unique_key_index_cache_policy_name = server_settings[ServerSetting::unique_key_index_cache_policy]; size_t unique_key_index_cache_size = server_settings[ServerSetting::unique_key_index_cache_size_bytes]; double unique_key_index_cache_size_ratio = server_settings[ServerSetting::unique_key_index_cache_size_ratio]; @@ -3123,6 +3137,7 @@ try global_context->updateUncompressedCacheConfiguration(config(), max_cache_size_in_bytes); global_context->updateMarkCacheConfiguration(config(), max_cache_size_in_bytes); + global_context->updateObjectStorageIdentityCacheConfiguration(config(), max_cache_size_in_bytes); global_context->updateUniqueKeyIndexCacheConfiguration(config(), max_cache_size_in_bytes); global_context->updateDeleteBitmapCacheConfiguration(config(), max_cache_size_in_bytes); global_context->updatePrimaryIndexCacheConfiguration(config(), max_cache_size_in_bytes); diff --git a/src/Access/Common/AccessType.h b/src/Access/Common/AccessType.h index 1df7f90de63e7..d79f2b43dcbf2 100644 --- a/src/Access/Common/AccessType.h +++ b/src/Access/Common/AccessType.h @@ -332,6 +332,7 @@ enum class AccessType : uint16_t M(SYSTEM_DROP_PAIMON_METADATA_CACHE, "SYSTEM CLEAR PAIMON_METADATA_CACHE, SYSTEM DROP PAIMON_METADATA_CACHE", GLOBAL, SYSTEM_DROP_CACHE) \ M(SYSTEM_DROP_AVRO_SCHEMA_CACHE, "SYSTEM CLEAR AVRO SCHEMA CACHE, SYSTEM DROP AVRO SCHEMA CACHE, DROP AVRO SCHEMA CACHE", GLOBAL, SYSTEM_DROP_CACHE) \ M(SYSTEM_DROP_PARQUET_METADATA_CACHE, "SYSTEM DROP PARQUET_METADATA_CACHE", GLOBAL, SYSTEM_DROP_CACHE) \ + M(SYSTEM_DROP_OBJECT_STORAGE_IDENTITY_CACHE, "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE, SYSTEM DROP OBJECT STORAGE IDENTITY CACHE", GLOBAL, SYSTEM_DROP_CACHE) \ M(SYSTEM_DROP_POINT_IN_POLYGON_CACHE, "SYSTEM CLEAR POINT IN POLYGON CACHE, SYSTEM DROP POINT IN POLYGON CACHE, DROP POINT IN POLYGON CACHE", GLOBAL, SYSTEM_DROP_CACHE) \ M(SYSTEM_DROP_PUFFIN_FILES_CACHE, "SYSTEM DROP PUFFIN_FILES_CACHE", GLOBAL, SYSTEM_DROP_CACHE) \ M(SYSTEM_PREWARM_PRIMARY_INDEX_CACHE, "SYSTEM PREWARM PRIMARY INDEX, PREWARM PRIMARY INDEX CACHE, PREWARM PRIMARY INDEX", GLOBAL, SYSTEM_DROP_CACHE) \ diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp index de465d12106ef..5d24cfa3bdacc 100644 --- a/src/Backups/BackupIO_S3.cpp +++ b/src/Backups/BackupIO_S3.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include @@ -649,6 +650,15 @@ void BackupWriterS3::copyFileFromDisk( const size_t source_size = ca_plan ? ca_plan->object.bytes_size : (copy_encrypted ? src_disk->getEncryptedFileSize(src_path) : src_disk->getFileSize(src_path)); const bool whole_object = (src_offset == 0) && (length == source_size); + const String dest_key = fs::path(s3_uri.key) / path_in_backup; + /// Runs when the copy throws after `CopyObject`, `PutObject`, or `CompleteMultipartUpload` + /// was sent. The success path below invalidates only if the copy returns. + const String endpoint = s3_uri.endpoint; + const String bucket = s3_uri.bucket; + auto on_object_creation_failure = [endpoint, bucket, dest_key]() + { + invalidateObjectStorageIdentityCache(endpoint, bucket, dest_key); + }; if (whole_object) { @@ -660,12 +670,16 @@ void BackupWriterS3::copyFileFromDisk( length, /* dest_s3_client */ client, /* dest_bucket */ s3_uri.bucket, - /* dest_key */ fs::path(s3_uri.key) / path_in_backup, + /* dest_key */ dest_key, s3_settings.request_settings, read_settings, blob_storage_log, runner, - create_read_buffer); + create_read_buffer, + /* object_metadata= */ std::nullopt, + ObjectStorageCopyMode::Default, + /* object_creation_client= */ nullptr, + on_object_creation_failure); } else { @@ -678,14 +692,18 @@ void BackupWriterS3::copyFileFromDisk( /* src_object_size= */ source_size, /* dest_s3_client */ client, /* dest_bucket */ s3_uri.bucket, - /* dest_key */ fs::path(s3_uri.key) / path_in_backup, + /* dest_key */ dest_key, s3_settings.request_settings, read_settings, blob_storage_log, runner, - create_read_buffer); + create_read_buffer, + /* object_metadata= */ std::nullopt, + on_object_creation_failure); } + /// The copy has replaced `dest_key`. A read that cached the previous ETag must not send it as `If-Match`. + invalidateObjectStorageIdentityCache(s3_uri.endpoint, s3_uri.bucket, dest_key); return; /// copied! } } @@ -699,6 +717,9 @@ void BackupWriterS3::copyFile(const String & destination, const String & source, LOG_TRACE(log, "Copying file inside backup from {} to {}", source, destination); const auto source_key = fs::path(s3_uri.key) / source; + const String dest_key = fs::path(s3_uri.key) / destination; + const String endpoint = s3_uri.endpoint; + const String bucket = s3_uri.bucket; copyS3File( client, /* src_bucket */ s3_uri.bucket, @@ -706,7 +727,7 @@ void BackupWriterS3::copyFile(const String & destination, const String & source, size, /* dest_s3_client= */ client, /* dest_bucket= */ s3_uri.bucket, - /* dest_key= */ fs::path(s3_uri.key) / destination, + /* dest_key= */ dest_key, s3_settings.request_settings, read_settings, blob_storage_log, @@ -716,14 +737,31 @@ void BackupWriterS3::copyFile(const String & destination, const String & source, LOG_TRACE(log, "Falling back to copy file inside backup from {} to {} through direct buffers", source, destination); return std::make_unique( client, s3_uri.bucket, source_key, s3_uri.version_id, s3_settings.request_settings, read_settings); + }, + /* object_metadata= */ std::nullopt, + ObjectStorageCopyMode::Default, + /* object_creation_client= */ nullptr, + [endpoint, bucket, dest_key]() + { + invalidateObjectStorageIdentityCache(endpoint, bucket, dest_key); }); + invalidateObjectStorageIdentityCache(s3_uri.endpoint, s3_uri.bucket, dest_key); } void BackupWriterS3::copyDataToFile(const String & path_in_backup, const CreateReadBufferFunction & create_read_buffer, UInt64 start_pos, UInt64 length) { - copyDataToS3File(create_read_buffer, start_pos, length, client, s3_uri.bucket, fs::path(s3_uri.key) / path_in_backup, + const String dest_key = fs::path(s3_uri.key) / path_in_backup; + const String endpoint = s3_uri.endpoint; + const String bucket = s3_uri.bucket; + copyDataToS3File(create_read_buffer, start_pos, length, client, s3_uri.bucket, dest_key, s3_settings.request_settings, blob_storage_log, - threadPoolCallbackRunnerUnsafe(getBackupsIOThreadPool().get(), ThreadName::S3_BACKUP_WRITER)); + threadPoolCallbackRunnerUnsafe(getBackupsIOThreadPool().get(), ThreadName::S3_BACKUP_WRITER), + /* object_metadata= */ std::nullopt, + [endpoint, bucket, dest_key]() + { + invalidateObjectStorageIdentityCache(endpoint, bucket, dest_key); + }); + invalidateObjectStorageIdentityCache(s3_uri.endpoint, s3_uri.bucket, dest_key); } BackupWriterS3::~BackupWriterS3() = default; @@ -752,38 +790,57 @@ std::unique_ptr BackupWriterS3::readFile(const String & file_name, s std::unique_ptr BackupWriterS3::writeFile(const String & file_name) { + /// Captured by value. The buffer can outlive this writer, and the cache itself is process-global. + const String endpoint = s3_uri.endpoint; + const String bucket = s3_uri.bucket; + const String object_key = fs::path(s3_uri.key) / file_name; return std::make_unique( client, - s3_uri.bucket, - fs::path(s3_uri.key) / file_name, + bucket, + object_key, DBMS_DEFAULT_BUFFER_SIZE, s3_settings.request_settings, blob_storage_log, std::nullopt, threadPoolCallbackRunnerUnsafe(getBackupsIOThreadPool().get(), ThreadName::S3_BACKUP_WRITER), - write_settings); + write_settings, + /* object_creation_client */ nullptr, + [endpoint, bucket, object_key]() + { + invalidateObjectStorageIdentityCache(endpoint, bucket, object_key); + }); } std::unique_ptr BackupWriterS3::writeFileIfNotExists(const String & file_name) { WriteSettings conditional_write_settings = write_settings; conditional_write_settings.object_storage_write_if_none_match = "*"; + const String endpoint = s3_uri.endpoint; + const String bucket = s3_uri.bucket; + const String object_key = fs::path(s3_uri.key) / file_name; return std::make_unique( client, - s3_uri.bucket, - fs::path(s3_uri.key) / file_name, + bucket, + object_key, DBMS_DEFAULT_BUFFER_SIZE, s3_settings.request_settings, blob_storage_log, std::nullopt, threadPoolCallbackRunnerUnsafe(getBackupsIOThreadPool().get(), ThreadName::S3_BACKUP_WRITER), - conditional_write_settings); + conditional_write_settings, + /* object_creation_client */ nullptr, + [endpoint, bucket, object_key]() + { + invalidateObjectStorageIdentityCache(endpoint, bucket, object_key); + }); } void BackupWriterS3::removeFile(const String & file_name) { - deleteFileFromS3(client, s3_uri.bucket, fs::path(s3_uri.key) / file_name, /* if_exists = */ true, + const String object_key = fs::path(s3_uri.key) / file_name; + deleteFileFromS3(client, s3_uri.bucket, object_key, /* if_exists = */ true, blob_storage_log); + invalidateObjectStorageIdentityCache(s3_uri.endpoint, s3_uri.bucket, object_key); } void BackupWriterS3::removeFiles(const Strings & file_names) @@ -798,6 +855,9 @@ void BackupWriterS3::removeFiles(const Strings & file_names) deleteFilesFromS3(client, s3_uri.bucket, keys, /* if_exists = */ true, s3_capabilities, batch_size, blob_storage_log); + /// A key that was deleted, or was already gone, must not keep its previous size and ETag. + for (const auto & key : keys) + invalidateObjectStorageIdentityCache(s3_uri.endpoint, s3_uri.bucket, key); } } diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index 5035b8cac7f33..35127b5c0289a 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -386,6 +386,8 @@ M(PaimonMetadataFilesCacheFiles, "Number of cached files in the Paimon metadata cache") \ M(ParquetMetadataCacheBytes, "Size of the Parquet metadata cache in bytes") \ M(ParquetMetadataCacheFiles, "Number of cached files in the Parquet metadata cache") \ + M(ObjectStorageIdentityCacheBytes, "Size of the object-storage identity cache in bytes") \ + M(ObjectStorageIdentityCacheCells, "Number of entries in the object-storage identity cache") \ M(PuffinFilesCacheBytes, "Size of the Puffin files cache in bytes (deletion vectors plus memoized footers)") \ M(PuffinFilesCacheFiles, "Number of cached entries in the Puffin files cache (deletion vectors plus memoized footers)") \ M(AvroSchemaCacheBytes, "Size of the Avro schema cache in bytes") \ diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp index 4afe507ca35c2..c7c49cfe8c718 100644 --- a/src/Common/FailPoint.cpp +++ b/src/Common/FailPoint.cpp @@ -84,6 +84,7 @@ static struct InitFiu ONCE(s3_read_buffer_throw_expired_token) \ ONCE(s3_send_request_throw_expired_token) \ REGULAR(s3_read_inject_etag_mismatch) \ + PAUSEABLE_ONCE(object_storage_identity_cache_pause_before_store) \ REGULAR(azure_inject_forbidden_response) \ ONCE(azure_inject_forbidden_response_once) \ REGULAR(azure_inject_auth_failure_on_request) \ diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 519c938c468f1..63947e7773d62 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -820,6 +820,8 @@ The server successfully detected this situation and will download merged part fr M(S3ListObjects, "Number of S3 API ListObjects calls.", ValueType::Number) \ M(S3ListObjectsMicroseconds, "Time of S3 API ListObjects execution.", ValueType::Microseconds) \ M(S3HeadObject, "Number of S3 API HeadObject calls.", ValueType::Number) \ + M(ObjectStorageIdentityCacheHits, "Number of object-storage identity (size/etag/Last-Modified) cache hits, avoiding a HEAD.", ValueType::Number) \ + M(ObjectStorageIdentityCacheMisses, "Number of object-storage identity cache misses that triggered a metadata request.", ValueType::Number) \ /* CA per-namespace S3 op instrumentation: one event per (namespace, op) cell of the table in */ \ /* CasInstrumentedBackend.cpp; attributes PUTs/HEADs/404s/412s/LISTs to the CA operation type. */ \ M(CASBlobPut, "Number of CAS blob PUT requests. Grows with blob uploads and indicates storage write traffic.", ValueType::Number) \ diff --git a/src/Common/QueryFuzzer.cpp b/src/Common/QueryFuzzer.cpp index fa45aaeb1a3ab..85a6045b814ee 100644 --- a/src/Common/QueryFuzzer.cpp +++ b/src/Common/QueryFuzzer.cpp @@ -7934,6 +7934,7 @@ void QueryFuzzer::fuzz(ASTPtr & ast) Type::CLEAR_ENCRYPTION_HEADERS_CACHE, Type::CLEAR_PAIMON_METADATA_CACHE, Type::CLEAR_PARQUET_METADATA_CACHE, + Type::CLEAR_OBJECT_STORAGE_IDENTITY_CACHE, Type::CLEAR_POINT_IN_POLYGON_CACHE, /// `CLEAR_{DISK_METADATA,DISTRIBUTED,FILESYSTEM,FORMAT_SCHEMA,QUERY,SCHEMA}_CACHE` /// are deliberately absent: they carry an operand this rotation would not preserve. diff --git a/src/Core/ServerSettings.cpp b/src/Core/ServerSettings.cpp index 26761d3f03ed1..d97924130c8f8 100644 --- a/src/Core/ServerSettings.cpp +++ b/src/Core/ServerSettings.cpp @@ -632,6 +632,9 @@ This setting can be modified at runtime and will take effect immediately. )", 0) \ DECLARE(Double, mark_cache_size_ratio, DEFAULT_MARK_CACHE_SIZE_RATIO, R"(The size of the protected queue (in case of SLRU policy) in the mark cache relative to the cache's total size.)", 0) \ DECLARE(Double, mark_cache_prewarm_ratio, 0.95, R"(The ratio of total size of mark cache to fill during prewarm.)", 0) \ + DECLARE(String, object_storage_identity_cache_policy, "SLRU", R"(Object-storage identity cache policy name (SLRU or LRU).)", 0) \ + DECLARE(UInt64, object_storage_identity_cache_size, 67108864, R"(Maximum size of the object-storage identity cache in bytes. This cache stores per-object size, ETag and Last-Modified so that opening an object does not require a fresh HEAD on every read. Set to 0 to disable. A reload (`SYSTEM RELOAD CONFIG`) with a non-zero size creates the cache when startup left it disabled, so a restart is not required to turn it on.)", 0) \ + DECLARE(Double, object_storage_identity_cache_size_ratio, 0.5, R"(The size of the protected queue (in case of SLRU policy) in the object-storage identity cache relative to the cache's total size.)", 0) \ DECLARE(String, unique_key_index_cache_policy, "SLRU", R"(UNIQUE KEY index cache policy name (SLRU or LRU).)", 0) \ DECLARE(UInt64, unique_key_index_cache_size_bytes, 1_GiB, R"(Maximum size (bytes) of the in-process cache for UNIQUE KEY index (SST) blocks. Set to 0 to disable the cache.)", 0) \ DECLARE(Double, unique_key_index_cache_size_ratio, 0.5, R"(The size of the protected queue (in case of SLRU policy) in the UNIQUE KEY index cache relative to the cache's total size.)", 0) \ diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h b/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h index d9d17af0f2eda..fd9b9a3e7dbb9 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -138,6 +139,19 @@ struct ObjectMetadata /// default epoch would make the schema/count caches treat the object as older than any cached entry and /// silently reuse a stale value. Cache validators must skip the cache when this is `false`. bool is_last_modified_known = true; + /// False when `last_modified` was reused from the identity cache. `_time` may still show it, but it was + /// not observed by this request. An external replace leaves that stored time older than the schema and + /// count registration time, so those caches would keep the previous schema and row count. + bool is_last_modified_fresh = true; + /// True when size and ETag were reused from the identity cache rather than observed by this request. + /// A filesystem or page cache keyed by that ETag can then serve the previous bytes without a GET. + bool is_from_identity_cache = false; + + /// Schema and count caches may use `last_modified` only when this request observed it. + bool lastModifiedConfirmsUnchanged() const + { + return is_last_modified_known && is_last_modified_fresh; + } std::string etag; /// Whether `etag` is a strong content/version identifier, i.e. different content is /// guaranteed to produce a different `etag`. Real S3/Azure ETags and the sub-second @@ -358,6 +372,14 @@ class IObjectStorage return getObjectMetadata(object.getPath(), with_tags); } + /// Same as `getObjectMetadata`, but do not answer from the identity cache. A content cache (filesystem + /// cache, page cache) keyed by a cached ETag would otherwise serve a replaced object's old bytes, because + /// a full hit never issues the GET that compares ETags. The default has no identity cache. + virtual ObjectMetadata getObjectMetadataIgnoringIdentityCache(const RelativePathWithMetadata & object, bool with_tags) const + { + return getObjectMetadata(object, with_tags); + } + /// Same as getObjectMetadata(), but ignores if object does not exist. virtual std::optional tryGetObjectMetadata(const std::string & path, bool with_tags) const = 0; virtual std::optional tryGetObjectMetadata(const RelativePathWithMetadata & object, bool with_tags) const diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp new file mode 100644 index 0000000000000..0b7f6988281e9 --- /dev/null +++ b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.cpp @@ -0,0 +1,90 @@ +#include + +#include +#include +#include + +namespace ProfileEvents +{ + extern const Event ObjectStorageIdentityCacheHits; + extern const Event ObjectStorageIdentityCacheMisses; +} + +namespace CurrentMetrics +{ + extern const Metric ObjectStorageIdentityCacheBytes; + extern const Metric ObjectStorageIdentityCacheCells; +} + +namespace DB +{ + +void invalidateObjectStorageIdentityCache(std::string_view endpoint, std::string_view bucket, std::string_view path) +{ + auto global_context = Context::getGlobalContextInstance(); + if (!global_context) + return; + + auto cache = global_context->getObjectStorageIdentityCache(); + if (!cache) + return; + + cache->invalidate(makeObjectStorageIdentityCacheKey(endpoint, bucket, path)); +} + +ObjectStorageIdentityCache::ObjectStorageIdentityCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio) + : Base( + cache_policy, + CurrentMetrics::ObjectStorageIdentityCacheBytes, + CurrentMetrics::ObjectStorageIdentityCacheCells, + max_size_in_bytes, + /*max_count*/ 0, + size_ratio) +{ +} + +std::optional ObjectStorageIdentityCache::tryGet(const String & key, UInt64 credential_hash) +{ + if (auto mapped = Base::get(key); mapped && mapped->credential_hash == credential_hash) + { + ProfileEvents::increment(ProfileEvents::ObjectStorageIdentityCacheHits); + return *mapped; + } + + ProfileEvents::increment(ProfileEvents::ObjectStorageIdentityCacheMisses); + return std::nullopt; +} + +ObjectStorageIdentityCache::Read ObjectStorageIdentityCache::beginFetch() +{ + std::lock_guard lock(epoch_mutex); + return Read{epoch, ++next_fetch_seq}; +} + +void ObjectStorageIdentityCache::setIfEpoch(const String & key, ObjectStorageIdentity identity, Read read) +{ + std::lock_guard lock(epoch_mutex); + if (epoch != read.epoch) + return; + /// `Base::get` takes the cache mutex after `epoch_mutex`, the same order as `set` and `remove`. + if (auto existing = Base::get(key); existing && existing->fetch_seq > read.seq) + return; + identity.fetch_seq = read.seq; + Base::set(key, std::make_shared(std::move(identity))); +} + +void ObjectStorageIdentityCache::invalidate(const String & key) +{ + std::lock_guard lock(epoch_mutex); + ++epoch; + Base::remove(key); +} + +void ObjectStorageIdentityCache::clear() +{ + std::lock_guard lock(epoch_mutex); + ++epoch; + Base::clear(); +} + +} diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h new file mode 100644 index 0000000000000..c57eb2ad29fff --- /dev/null +++ b/src/Disks/DiskObjectStorage/ObjectStorages/ObjectStorageIdentityCache.h @@ -0,0 +1,129 @@ +#pragma once + +#include +#include + +#include +#include +#include + +namespace DB +{ + +/// Object-storage "identity": the bits needed to open an object without a fresh HEAD - its size, +/// ETag and Last-Modified. +/// +/// It exists to break the ordering that forces a HEAD before every open: object caches key on +/// (path, etag), so the etag must be known up front. Here the key is the store plus the path +/// (`makeObjectStorageIdentityCacheKey`) and the etag is stored in the value, learned once (via a +/// HEAD or a GET response) and reused. Every entry comes from a HEAD, which always returns +/// Last-Modified, so `is_last_modified_known` only means "the object has a modification time". +struct ObjectStorageIdentity +{ + String etag; + UInt64 size = 0; + bool is_size_known = true; + /// Seconds since epoch. Meaningful only when `is_last_modified_known`. + Int64 last_modified = 0; + bool is_last_modified_known = false; + /// Orders stores of this key. A fetch that started earlier must not overwrite one that started later. + /// Not part of the object's identity. + UInt64 fetch_seq = 0; + /// SipHash of the access key, secret, and session token that signed the HEAD. Not a secret. + /// A hit is only valid for the same principal: the STS mock used in tests returns one access key + /// with a different secret per role session, so the access key alone does not separate them. + UInt64 credential_hash = 0; + + size_t memoryUsage() const + { + return sizeof(ObjectStorageIdentity) + etag.capacity(); + } +}; + +/// Approximate per-entry weight (bytes) for size-bounded eviction. +struct ObjectStorageIdentityWeight +{ + /// Extra bytes spent on the key string, hashmap node, list links, shared_ptr, etc. + static constexpr size_t OVERHEAD = 128; + + size_t operator()(const ObjectStorageIdentity & identity) const + { + return identity.memoryUsage() + OVERHEAD; + } +}; + +/// One object on one store. Bucket and path are not enough: two S3-compatible endpoints can reuse +/// the same bucket name. Fields are separated by NUL so a slash in the endpoint cannot alias another field. +inline String makeObjectStorageIdentityCacheKey(std::string_view endpoint, std::string_view bucket, std::string_view path) +{ + String key; + key.reserve(endpoint.size() + bucket.size() + path.size() + 2); + key.append(endpoint); + key.push_back('\0'); + key.append(bucket); + key.push_back('\0'); + key.append(path); + return key; +} + +/// Drop the identity cached for this endpoint, bucket, and path. No-op when the global context or the +/// cache is not set. The next read of the path requests the metadata again. +void invalidateObjectStorageIdentityCache(std::string_view endpoint, std::string_view bucket, std::string_view path); + +/// Cache of object-storage identity, keyed by endpoint, bucket and object path, so opening an object +/// does not require a fresh HEAD on every read (a lake scan otherwise re-HEADs +/// every candidate file on every query). Backed by CacheBase (LRU/SLRU) for size-bounded eviction +/// and thread-safety. +/// +/// It is owned by the global Context (see Context::getObjectStorageIdentityCache), sized from server +/// settings, and cleared by `SYSTEM DROP OBJECT STORAGE IDENTITY CACHE`. A write, copy, or delete by +/// this server, and a read that finds the object replaced or missing, removes the entry and advances +/// the epoch. A metadata fetch takes `beginFetch` before its request and stores its result only if +/// that epoch is unchanged and no fetch of the same key started later, so an older observation cannot +/// put a stale identity back. The next read of that key requests the metadata again. +/// Low-level object-storage code reaches the cache via Context::getGlobalContextInstance() and must +/// tolerate a null pointer (no global context yet) by simply issuing the metadata request uncached. +class ObjectStorageIdentityCache + : public CacheBase, ObjectStorageIdentityWeight> +{ + using Base = CacheBase, ObjectStorageIdentityWeight>; + +public: + ObjectStorageIdentityCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio); + + /// Look up an identity by `makeObjectStorageIdentityCacheKey` for the principal `credential_hash`. + /// An entry learned by another principal is a miss and is left in place. Increments the hit/miss ProfileEvents. + std::optional tryGet(const String & key, UInt64 credential_hash); + + /// Epoch at the start of a metadata fetch, and a sequence that orders that fetch against others. + struct Read + { + UInt64 epoch = 0; + UInt64 seq = 0; + }; + + /// Call before the metadata request and pass the result to `setIfEpoch`. + Read beginFetch(); + + /// Store `identity` only when `read.epoch` is still current and no later fetch of `key` has stored. + void setIfEpoch(const String & key, ObjectStorageIdentity identity, Read read); + + /// Advance the epoch and drop the entry. A fetch already in flight will not put the old identity back. + void invalidate(const String & key); + + /// Drop every entry and advance the epoch. Hides `CacheBase::clear` so a fetch in flight across the + /// drop cannot store afterwards. + void clear(); + +private: + mutable std::mutex epoch_mutex; + /// Advanced by every `invalidate` and `clear`. Guarded by `epoch_mutex`. It is never reset, so a + /// fetch in flight can never match a later epoch by accident. + UInt64 epoch = 0; + /// Advanced by every `beginFetch`. Guarded by `epoch_mutex`. + UInt64 next_fetch_seq = 0; +}; + +using ObjectStorageIdentityCachePtr = std::shared_ptr; + +} diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp index 41c3ba97b0b47..359576e0b5ff6 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp @@ -19,6 +19,8 @@ #include #include #include +#include +#include #include #include #include @@ -33,6 +35,7 @@ #include #include +#include #include #include #include @@ -61,14 +64,33 @@ namespace CurrentMetrics namespace DB::FailPoints { extern const char object_storage_force_refresh_callback_success[]; + extern const char object_storage_identity_cache_pause_before_store[]; } namespace DB { + +/// Distinguishes principals that sign with the same access key and a different secret or session token. +static UInt64 s3CredentialHash(const S3::Client & client) +{ + const auto credentials = client.getCredentials(); + SipHash hash; + const auto update_field = [&](const auto & field) + { + hash.update(field.data(), field.size()); + hash.update("\0", 1); + }; + update_field(credentials.GetAWSAccessKeyId()); + update_field(credentials.GetAWSSecretKey()); + update_field(credentials.GetSessionToken()); + return hash.get64(); +} + namespace Setting { extern const SettingsBool s3_validate_request_settings; + extern const SettingsBool s3_validate_etag_on_read; } namespace S3RequestSetting @@ -380,6 +402,13 @@ std::unique_ptr S3ObjectStorage::readObject( /// NOLINT } } + /// The buffer can outlive this storage, so the callback keeps its own copy of the key parts. + /// A GET that finds the object replaced or missing drops the cached identity, and the next read + /// requests the metadata again instead of reusing it. + const String endpoint = uri.endpoint; + const String bucket = uri.bucket; + const String path = object.remote_path; + return std::make_unique( clientForRetryProfile(ObjectStorageControlRequest{ .profile = read_settings.object_storage_retry_profile, @@ -401,7 +430,11 @@ std::unique_ptr S3ObjectStorage::readObject( /// NOLINT (object.bytes_size && object.bytes_size != StoredObject::UnknownSize) ? std::optional(object.bytes_size) : std::nullopt, refresh_callback, std::move(blob_storage_log), - object.etag); + object.etag, + [endpoint, bucket, path]() + { + invalidateObjectStorageIdentityCache(endpoint, bucket, path); + }); } SmallObjectDataWithMetadata S3ObjectStorage::readSmallObjectAndGetObjectMetadata( /// NOLINT @@ -481,6 +514,11 @@ std::unique_ptr S3ObjectStorage::writeObject( /// NOLIN .attempt_timeout_ms = write_settings.object_storage_attempt_timeout_ms, .connect_timeout_cap_ms = write_settings.object_storage_connect_timeout_cap_ms}); + /// Captured by value. The buffer can outlive this storage, and the cache itself is process-global. + const String endpoint = uri.endpoint; + const String bucket = uri.bucket; + const String path = object.remote_path; + return std::make_unique( used_client, uri.bucket, @@ -491,7 +529,11 @@ std::unique_ptr S3ObjectStorage::writeObject( /// NOLIN attributes, std::move(scheduler), disk_write_settings, - objectCreationClient(write_settings)); + objectCreationClient(write_settings), + [endpoint, bucket, path]() + { + invalidateObjectStorageIdentityCache(endpoint, bucket, path); + }); } @@ -587,6 +629,7 @@ void S3ObjectStorage::removeObjectImpl(const StoredObject & object, bool if_exis deleteFileFromS3(client.get(), uri.bucket, object.remote_path, if_exists, blob_storage_log, object.local_path, object.bytes_size, ProfileEvents::DiskS3DeleteObjects); + invalidateCachedIdentity(object.remote_path); } void S3ObjectStorage::removeObjectsImpl(const StoredObjects & objects, bool if_exists) @@ -616,6 +659,8 @@ void S3ObjectStorage::removeObjectsImpl(const StoredObjects & objects, bool if_e s3_capabilities, settings_ptr->request_settings[S3RequestSetting::objects_chunk_size_to_delete], blob_storage_log, local_paths_for_blob_storage_log, file_sizes_for_blob_storage_log, ProfileEvents::DiskS3DeleteObjects); + for (const auto & object : objects) + invalidateCachedIdentity(object.remote_path); } void S3ObjectStorage::removeObjectIfExists(const StoredObject & object) @@ -671,7 +716,10 @@ ConditionalRemoveResult S3ObjectStorage::removeObjectIfTokenMatchesImpl( outcome.IsSuccess() ? "" : outcome.GetError().GetMessage()); if (outcome.IsSuccess()) + { + invalidateCachedIdentity(object.remote_path); return {ConditionalRemoveOutcome::Removed, outcome.GetResult().GetDeleteMarker()}; + } const auto & err = outcome.GetError(); @@ -679,11 +727,18 @@ ConditionalRemoveResult S3ObjectStorage::removeObjectIfTokenMatchesImpl( /// (see `S3::isPreconditionFailedError` for the one policy). Callers treat 'mismatch' and 'gone' /// alike (re-validate); a genuine absence is disambiguated downstream by a HEAD re-check. if (S3::isPreconditionFailedError(err)) + { + /// The etag we held is not the object's etag, so the cached identity is not this generation. + invalidateCachedIdentity(object.remote_path); return {ConditionalRemoveOutcome::TokenMismatch, false}; + } /// The object no longer exists (404). Protocol callers treat 'mismatch' and 'gone' alike (re-validate). if (S3::isNotFoundError(err.GetErrorType())) + { + invalidateCachedIdentity(object.remote_path); return {ConditionalRemoveOutcome::NotFound, false}; + } throw S3Exception(err.GetErrorType(), "{} (Code: {}, S3 exception: '{}') while conditionally removing object with path {} from S3", @@ -715,6 +770,7 @@ void S3ObjectStorage::removeObjectsIfExistImpl( deleteFileFromS3(used_client, uri.bucket, object.remote_path, /*if_exists=*/ true, BlobStorageLogWriter::create(disk_name), object.local_path, object.bytes_size, ProfileEvents::DiskS3DeleteObjects, attempt_seed); + invalidateCachedIdentity(object.remote_path); return; } @@ -802,6 +858,21 @@ void S3ObjectStorage::removeObjectsIfExistImpl( first_error_name = err.GetCode(); } } + /// A key that was deleted, or was already gone, must not keep its previous size and ETag. + /// A key whose delete failed is still the object the cache describes. + for (const auto & object : objects) + { + bool delete_failed = false; + for (const auto & err : outcome.GetResult().GetErrors()) + { + if (err.GetKey() != object.remote_path) + continue; + if (!S3::isNotFoundError(classifyDeleteObjectsErrorCode(err.GetCode()))) + delete_failed = true; + } + if (!delete_failed) + invalidateCachedIdentity(object.remote_path); + } if (first_error_type) throw S3Exception( PreformattedMessage::create("batch removal left objects behind: [{}]", failed_keys), *first_error_type, first_error_name); @@ -979,11 +1050,90 @@ std::optional S3ObjectStorage::tryGetObjectMetadataImpl( ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool with_tags) const { + return getObjectMetadataImpl(path, with_tags, /*use_identity_cache=*/ true); +} + +ObjectMetadata S3ObjectStorage::getObjectMetadataImpl(const std::string & path, bool with_tags, bool use_identity_cache) const +{ + /// When the caller only needs identity (size / etag / Last-Modified, not tags or user metadata) we + /// take a fast path: consult the identity cache to skip the HEAD on repeat opens. A miss issues a + /// plain HEAD. This is what removes the per-file metadata request on lake scans. The tags path is unchanged. + /// See ObjectStorageIdentityCache. + /// + /// The cache is owned by the global Context and sized from server settings. Low-level + /// object-storage code can run before a global context exists; a null pointer means "no cache" + /// and the request is issued uncached. + const bool identity_only = !with_tags; + /// Bucket and path alone collide when two endpoints reuse a bucket name. + const String identity_key = makeObjectStorageIdentityCacheKey(uri.endpoint, uri.bucket, path); + ObjectStorageIdentityCachePtr identity_cache; + if (identity_only) + { + if (auto global_context = Context::getGlobalContextInstance()) + identity_cache = global_context->getObjectStorageIdentityCache(); + + /// A cached size is the right bound of a small-file prefetch and the file length for Parquet. + /// That is safe when the GET is pinned to the cached ETag. With `s3_validate_etag_on_read` off + /// (including `compatibility` before 26.7) nothing checks the ETag, so an external replace would + /// be read as the previous generation. Skip the hit and HEAD. The result is still stored, so a + /// later read that does check the ETag can hit. + bool consult_identity_cache = use_identity_cache; + if (consult_identity_cache) + { + if (auto query_context = CurrentThread::tryGetQueryContext()) + { + if (!query_context->getSettingsRef()[Setting::s3_validate_etag_on_read]) + consult_identity_cache = false; + } + } + + if (consult_identity_cache && identity_cache) + { + /// Computed only for a lookup. `getCredentials` resolves an assumed role, and a different + /// secret must not reuse this entry. The STS test mock returns one access key for every session. + const UInt64 credential_hash = s3CredentialHash(*client.get()); + if (auto cached = identity_cache->tryGet(identity_key, credential_hash)) + { + ObjectMetadata result; + result.size_bytes = cached->size; + result.is_size_known = cached->is_size_known; + result.etag = cached->etag; + result.is_from_identity_cache = true; + if (cached->is_last_modified_known) + { + result.last_modified = Poco::Timestamp::fromEpochTime(static_cast(cached->last_modified)); + /// Real enough for `_time`, but not a freshness proof. The schema and count caches + /// treat an older time as "unchanged" and would keep a replaced object's schema and row count. + result.is_last_modified_fresh = false; + } + else + result.is_last_modified_known = false; + return result; + } + } + } + + /// Captured before the request. `setIfEpoch` stores the result only while this epoch is unchanged + /// and no later fetch of the same key has stored, so a write, copy, delete, drop, or an older + /// observation cannot put a stale identity back. + ObjectStorageIdentityCache::Read identity_read; + if (identity_cache) + identity_read = identity_cache->beginFetch(); + auto settings_ptr = s3_settings.get(); + + auto fetch = [&](const S3::Client & c) -> S3::ObjectInfo + { + /// A plain HEAD returns size, ETag and Last-Modified, which is everything the identity cache stores. + if (identity_only) + return S3::getObjectInfo(c, uri.bucket, path, /*version_id=*/ {}, /*with_metadata=*/ false, /*with_tags=*/ false); + return S3::getObjectInfo(c, uri.bucket, path, /*version_id=*/ {}, /*with_metadata=*/ true, /*with_tags=*/ with_tags); + }; + S3::ObjectInfo object_info; try { - object_info = S3::getObjectInfo(*client.get(), uri.bucket, path, /*version_id=*/ {}, /*with_metadata=*/ true, /*with_tags=*/ with_tags); + object_info = fetch(*client.get()); } catch (DB::Exception & e) { @@ -994,7 +1144,7 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool if (new_client) { client.set(std::move(new_client)); - object_info = S3::getObjectInfo(*client.get(), uri.bucket, path, /*version_id=*/ {}, /*with_metadata=*/ true, /*with_tags=*/ with_tags); + object_info = fetch(*client.get()); updated = true; } } @@ -1012,6 +1162,26 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path, bool result.etag = object_info.etag; result.tags = std::move(object_info.tags); result.attributes = object_info.metadata; + /// Epoch 0 means the response had no Last-Modified. Keep it out of `_time` and the schema and count caches. + if (identity_only) + result.is_last_modified_known = object_info.last_modification_time != 0; + + if (identity_cache) + { + /// The read that reached this point already holds the identity from before any overwrite that + /// happened while it was paused. Resuming must not store that identity. + FailPointInjection::pauseFailPoint(FailPoints::object_storage_identity_cache_pause_before_store); + ObjectStorageIdentity identity; + identity.etag = result.etag; + identity.size = result.size_bytes; + identity.is_size_known = result.is_size_known; + identity.is_last_modified_known = result.is_last_modified_known; + if (identity.is_last_modified_known) + identity.last_modified = static_cast(result.last_modified.epochTime()); + /// After a credential refresh the client that signed the HEAD may not be the one looked up. + identity.credential_hash = s3CredentialHash(*client.get()); + identity_cache->setIfEpoch(identity_key, std::move(identity), identity_read); + } return result; } @@ -1055,7 +1225,12 @@ void S3ObjectStorage::copyObjectToAnotherObjectStorage( // NOLINT [&, this]{ return readObject(object_from, read_settings_to_use);}, object_to_attributes, write_settings.object_storage_copy_mode, - dest_s3->objectCreationClient(write_settings)); + dest_s3->objectCreationClient(write_settings), + [dest_s3, dest_path = object_to.remote_path]() + { + dest_s3->invalidateCachedIdentity(dest_path); + }); + dest_s3->invalidateCachedIdentity(object_to.remote_path); return; } catch (S3Exception & exc) @@ -1134,7 +1309,12 @@ void S3ObjectStorage::copyObject( // NOLINT [&, this]{ return readObject(object_from, read_settings_to_use);}, object_to_attributes, write_settings.object_storage_copy_mode, - objectCreationClient(write_settings)); + objectCreationClient(write_settings), + [this, dest_path = object_to.remote_path]() + { + invalidateCachedIdentity(dest_path); + }); + invalidateCachedIdentity(object_to.remote_path); } void S3ObjectStorage::shutdown() @@ -1347,6 +1527,11 @@ bool S3ObjectStorage::hasSingleAttemptClientForTest(uint64_t request_timeout_ms, return single_attempt_clients.contains(std::make_tuple(request_timeout_ms, connect_timeout_cap_ms, false)); } +void S3ObjectStorage::invalidateCachedIdentity(const std::string & path) const +{ + invalidateObjectStorageIdentityCache(uri.endpoint, uri.bucket, path); +} + std::shared_ptr S3ObjectStorage::clientForRetryProfile(const ObjectStorageControlRequest & request) const { /// getSingleAttemptClient is only invoked when actually selected, so an ordinary request never diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h index 2d285e289d9b0..99f1eaa950f70 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h @@ -147,6 +147,11 @@ class S3ObjectStorage : public IObjectStorage ObjectMetadata getObjectMetadata(const std::string & path, bool with_tags) const override; + ObjectMetadata getObjectMetadataIgnoringIdentityCache(const RelativePathWithMetadata & object, bool with_tags) const override + { + return getObjectMetadataImpl(object.getPath(), with_tags, /*use_identity_cache=*/ false); + } + std::optional tryGetObjectMetadata(const std::string & path, bool with_tags) const override; /// Marks the HEAD request eligible for the typed NativeConditional mode, so the CAS backend's @@ -248,6 +253,13 @@ class S3ObjectStorage : public IObjectStorage std::shared_ptr clientForRetryProfile(const ObjectStorageControlRequest & request) const; + /// Drop the identity-cache entry for `path`. The next read of this key requests the metadata again. + void invalidateCachedIdentity(const std::string & path) const; + + /// `use_identity_cache` false still stores the result. It only skips the lookup, so a content cache + /// can key itself on an ETag observed by this request. + ObjectMetadata getObjectMetadataImpl(const std::string & path, bool with_tags, bool use_identity_cache) const; + /// The single-attempt client for the object-creating requests of a write that asks for it, else null. std::shared_ptr objectCreationClient(const WriteSettings & write_settings) const; diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp index 3062b737b4566..2e4a0f1d7cebe 100644 --- a/src/IO/ReadBufferFromS3.cpp +++ b/src/IO/ReadBufferFromS3.cpp @@ -106,7 +106,8 @@ ReadBufferFromS3::ReadBufferFromS3( std::optional file_size_, const S3CredentialsRefreshCallback & credentials_refresh_callback_, BlobStorageLogWriterPtr blob_storage_log_, - const String & expected_etag_) + const String & expected_etag_, + std::function on_identity_rejected_) : ReadBufferFromFileBase() , client_ptr(std::move(client_ptr_)) , bucket(bucket_) @@ -121,6 +122,7 @@ ReadBufferFromS3::ReadBufferFromS3( , restricted_seek(restricted_seek_) , credentials_refresh_callback(credentials_refresh_callback_) , blob_storage_log(std::move(blob_storage_log_)) + , on_identity_rejected(std::move(on_identity_rejected_)) { file_size = file_size_; } @@ -669,11 +671,14 @@ Aws::S3::Model::GetObjectResult ReadBufferFromS3::sendRequest(size_t attempt, si /// on ETag drift. Skip ?versionId= reads (pinned to an immutable version that expected_etag, /// taken from the current version, would falsely mismatch) and empty response ETags. if (version_id.empty() && !expected_etag.empty() && !response_etag.empty() && response_etag != expected_etag) + { + rejectCachedIdentity(); throw Exception( ErrorCodes::S3_OBJECT_CHANGED_DURING_READ, "S3 object {}/{} was replaced during read (etag changed from {} to {}); " "retry the query, or set s3_validate_etag_on_read=0 to disable this check", bucket, key, expected_etag, response_etag); + } if (blob_storage_log) { @@ -705,15 +710,28 @@ Aws::S3::Model::GetObjectResult ReadBufferFromS3::sendRequest(size_t attempt, si /// error: S3Exception keeps only the Aws S3Errors code, not the HTTP 412 status. if (version_id.empty() && !expected_etag.empty() && error.GetResponseCode() == Aws::Http::HttpResponseCode::PRECONDITION_FAILED) + { + rejectCachedIdentity(); throw Exception( ErrorCodes::S3_OBJECT_CHANGED_DURING_READ, "S3 object {}/{} was replaced during read (If-Match on etag {} failed); " "retry the query, or set s3_validate_etag_on_read=0 to disable this check", bucket, key, expected_etag); + } + + /// A cached size and ETag for a key that is gone would fail the same way on every retry. + if (version_id.empty() && S3::isNotFoundError(error.GetErrorType())) + rejectCachedIdentity(); throw S3Exception(error.GetMessage(), error.GetErrorType()); } +void ReadBufferFromS3::rejectCachedIdentity() const +{ + if (on_identity_rejected) + on_identity_rejected(); +} + ObjectMetadata ReadBufferFromS3::getObjectMetadataFromTheLastRequest() const { if (!impl) diff --git a/src/IO/ReadBufferFromS3.h b/src/IO/ReadBufferFromS3.h index d99c61f015a72..dff4216827b2a 100644 --- a/src/IO/ReadBufferFromS3.h +++ b/src/IO/ReadBufferFromS3.h @@ -65,7 +65,8 @@ class ReadBufferFromS3 : public ReadBufferFromFileBase std::optional file_size = std::nullopt, const S3CredentialsRefreshCallback & credentials_refresh_callback_ = [] {return nullptr;}, BlobStorageLogWriterPtr blob_storage_log_ = {}, - const String & expected_etag_ = {} + const String & expected_etag_ = {}, + std::function on_identity_rejected_ = {} ); ~ReadBufferFromS3() override = default; @@ -125,6 +126,9 @@ class ReadBufferFromS3 : public ReadBufferFromFileBase Aws::S3::Model::GetObjectResult sendRequest(size_t attempt, size_t range_begin, std::optional range_end_incl) const; + /// The cached identity of this key is not the object that was read. Drop it so a retry fetches again. + void rejectCachedIdentity() const; + /// Drops the identity baseline. Called when the next request is a reissue for a range the caller /// explicitly repositioned to (seek, or a change of the read-until bound), as opposed to a retry of /// the same range after a failure: the bytes already delivered before the reposition reached the @@ -159,6 +163,9 @@ class ReadBufferFromS3 : public ReadBufferFromFileBase const S3CredentialsRefreshCallback credentials_refresh_callback; mutable BlobStorageLogWriterPtr blob_storage_log; + + /// Called when a GET finds this key replaced or missing. Empty when the caller has no identity cache. + std::function on_identity_rejected; }; } diff --git a/src/IO/S3/copyS3File.cpp b/src/IO/S3/copyS3File.cpp index 76f2dbb2df575..10db4a450f146 100644 --- a/src/IO/S3/copyS3File.cpp +++ b/src/IO/S3/copyS3File.cpp @@ -92,7 +92,8 @@ namespace ThreadPoolCallbackRunnerUnsafe schedule_, BlobStorageLogWriterPtr blob_storage_log_, const LoggerPtr log_, - const std::shared_ptr & object_creation_client_ = nullptr) + const std::shared_ptr & object_creation_client_ = nullptr, + OnObjectCreationFailure on_object_creation_failure_ = {}) : client_ptr(client_ptr_) , object_creation_client(object_creation_client_ ? object_creation_client_ : client_ptr_) , dest_bucket(dest_bucket_) @@ -102,6 +103,7 @@ namespace , schedule(schedule_) , blob_storage_log(blob_storage_log_) , log(log_) + , on_object_creation_failure(std::move(on_object_creation_failure_)) , num_parts(0) , normal_part_size(0) { @@ -120,6 +122,11 @@ namespace ThreadPoolCallbackRunnerUnsafe schedule; BlobStorageLogWriterPtr blob_storage_log; const LoggerPtr log; + /// Called from `performCopy` when this copy throws after an object-creating request was sent. + OnObjectCreationFailure on_object_creation_failure; + /// Set immediately before `CopyObject`, `PutObject`, or `CompleteMultipartUpload`. A lost + /// response can still have committed the destination. Read on the same thread that writes it. + bool object_creation_request_sent = false; /// Represents a task uploading a single part. /// Keep this struct small because there can be thousands of parts. @@ -139,6 +146,12 @@ namespace std::atomic num_finished_parts = 0; std::atomic has_failed = false; + void notifyObjectCreationFailure() + { + if (object_creation_request_sent && on_object_creation_failure) + on_object_creation_failure(); + } + void fillCreateMultipartRequest(S3::CreateMultipartUploadRequest & request) { request.SetBucket(dest_bucket); @@ -219,6 +232,9 @@ namespace ProfileEvents::increment(ProfileEvents::DiskS3CompleteMultipartUpload); Stopwatch watch; + /// Set before the call. A failure after S3 has committed the upload must still reach + /// `performCopy`, which drops the cached identity only once this request has been sent. + object_creation_request_sent = true; auto outcome = object_creation_client->CompleteMultipartUpload(request); auto elapsed = watch.elapsedMicroseconds(); @@ -450,8 +466,19 @@ namespace const S3::S3RequestSettings & request_settings_, const std::optional & object_metadata_, ThreadPoolCallbackRunnerUnsafe schedule_, - BlobStorageLogWriterPtr blob_storage_log_) - : UploadHelper(client_ptr_, dest_bucket_, dest_key_, request_settings_, object_metadata_, schedule_, blob_storage_log_, getLogger("copyDataToS3File")) + BlobStorageLogWriterPtr blob_storage_log_, + OnObjectCreationFailure on_object_creation_failure_) + : UploadHelper( + client_ptr_, + dest_bucket_, + dest_key_, + request_settings_, + object_metadata_, + schedule_, + blob_storage_log_, + getLogger("copyDataToS3File"), + /* object_creation_client_= */ nullptr, + std::move(on_object_creation_failure_)) , create_read_buffer(create_read_buffer_) , offset(offset_) , size(size_) @@ -460,13 +487,21 @@ namespace void performCopy() { - if (size <= request_settings[S3RequestSetting::max_single_part_upload_size]) - performSinglepartUpload(); - else - performMultipartUpload(); + try + { + if (size <= request_settings[S3RequestSetting::max_single_part_upload_size]) + performSinglepartUpload(); + else + performMultipartUpload(); - if (request_settings[S3RequestSetting::check_objects_after_upload]) - checkObjectAfterUpload(); + if (request_settings[S3RequestSetting::check_objects_after_upload]) + checkObjectAfterUpload(); + } + catch (...) + { + notifyObjectCreationFailure(); + throw; + } } private: @@ -521,6 +556,8 @@ namespace ProfileEvents::increment(ProfileEvents::DiskS3PutObject); Stopwatch watch; + /// Set before the call. A lost `PutObject` response can still have committed the object. + object_creation_request_sent = true; auto outcome = client_ptr->PutObject(request); auto elapsed = watch.elapsedMicroseconds(); @@ -647,7 +684,8 @@ namespace std::function fallback_method_, bool is_ranged_copy_, bool allow_fallback_, - const std::shared_ptr & object_creation_client_) + const std::shared_ptr & object_creation_client_, + OnObjectCreationFailure on_object_creation_failure_) : UploadHelper( client_ptr_, dest_bucket_, @@ -657,7 +695,8 @@ namespace schedule_, blob_storage_log_, getLogger("copyS3File"), - object_creation_client_) + object_creation_client_, + std::move(on_object_creation_failure_)) , src_bucket(src_bucket_) , src_key(src_key_) , offset(src_offset_) @@ -672,6 +711,19 @@ namespace } void performCopy() + { + try + { + performCopyImpl(); + } + catch (...) + { + notifyObjectCreationFailure(); + throw; + } + } + + void performCopyImpl() { LOG_TEST(log, "Copy object {} to {} using native copy", src_key, dest_key); @@ -759,6 +811,8 @@ namespace if (client_ptr->isClientForDisk()) ProfileEvents::increment(ProfileEvents::DiskS3CopyObject); + /// Set before the call. A lost `CopyObject` response can still have replaced the destination. + object_creation_request_sent = true; auto outcome = object_creation_client->CopyObject(request); if (outcome.IsSuccess()) { @@ -915,7 +969,8 @@ void copyDataToS3File( const S3::S3RequestSettings & settings, BlobStorageLogWriterPtr blob_storage_log, ThreadPoolCallbackRunnerUnsafe schedule, - const std::optional & object_metadata) + const std::optional & object_metadata, + OnObjectCreationFailure on_object_creation_failure) { CopyDataToFileHelper helper{ create_read_buffer, @@ -927,7 +982,8 @@ void copyDataToS3File( settings, object_metadata, schedule, - blob_storage_log}; + blob_storage_log, + std::move(on_object_creation_failure)}; helper.performCopy(); } @@ -954,12 +1010,14 @@ namespace const std::optional & object_metadata, bool is_ranged_copy, ObjectStorageCopyMode copy_mode, - std::shared_ptr object_creation_client) + std::shared_ptr object_creation_client, + OnObjectCreationFailure on_object_creation_failure) { if (!dest_s3_client) dest_s3_client = src_s3_client; - std::function fallback_method = [&] mutable + /// Copy the callback. The native helper takes the original, and this fallback can run after that move. + std::function fallback_method = [&, on_object_creation_failure]() mutable { copyDataToS3File( fallback_file_reader, @@ -971,7 +1029,8 @@ namespace settings, blob_storage_log, schedule, - object_metadata); + object_metadata, + on_object_creation_failure); }; if (!settings[S3RequestSetting::allow_native_copy]) @@ -1003,7 +1062,8 @@ namespace std::move(fallback_method), is_ranged_copy, /*allow_fallback=*/copy_mode == ObjectStorageCopyMode::Default, - object_creation_client}; + object_creation_client, + std::move(on_object_creation_failure)}; helper.performCopy(); } } @@ -1023,7 +1083,8 @@ void copyS3File( const CreateReadBuffer & fallback_file_reader, const std::optional & object_metadata, ObjectStorageCopyMode copy_mode, - std::shared_ptr object_creation_client) + std::shared_ptr object_creation_client, + OnObjectCreationFailure on_object_creation_failure) { /// The read-write fallback would write through the retrying client. chassert(!object_creation_client || copy_mode == ObjectStorageCopyMode::NativeOnly); @@ -1046,7 +1107,8 @@ void copyS3File( object_metadata, /* is_ranged_copy= */ false, copy_mode, - std::move(object_creation_client)); + std::move(object_creation_client), + std::move(on_object_creation_failure)); } void copyS3FileRange( @@ -1064,7 +1126,8 @@ void copyS3FileRange( BlobStorageLogWriterPtr blob_storage_log, ThreadPoolCallbackRunnerUnsafe schedule, const CreateReadBuffer & fallback_file_reader, - const std::optional & object_metadata) + const std::optional & object_metadata, + OnObjectCreationFailure on_object_creation_failure) { copyS3FileImpl( std::move(src_s3_client), @@ -1084,7 +1147,8 @@ void copyS3FileRange( object_metadata, /* is_ranged_copy= */ true, ObjectStorageCopyMode::Default, - /* object_creation_client= */ nullptr); + /* object_creation_client= */ nullptr, + std::move(on_object_creation_failure)); } } diff --git a/src/IO/S3/copyS3File.h b/src/IO/S3/copyS3File.h index 9106070471bab..1a3aef0e903fd 100644 --- a/src/IO/S3/copyS3File.h +++ b/src/IO/S3/copyS3File.h @@ -21,6 +21,12 @@ class StdStreamFromReadBuffer; using CreateReadBuffer = std::function()>; +/// Invoked when `copyS3File`, `copyS3FileRange`, or `copyDataToS3File` throws after `CopyObject`, +/// `PutObject`, or `CompleteMultipartUpload` was sent. That request may have committed the +/// destination. A failure before any of those requests does not call it. A successful copy does +/// not call it either: the caller drops a cached identity after the function returns. +using OnObjectCreationFailure = std::function; + /// Builds the S3 upload request body for the part [offset, offset + size) of the source. /// The part is read fully into memory up front, so the returned body has no failable inner /// source buffer: the AWS SDK can rewind (clear(); seekg(0)) and resend the body on a retry @@ -60,7 +66,8 @@ void copyS3File( const CreateReadBuffer & fallback_file_reader, const std::optional & object_metadata = std::nullopt, ObjectStorageCopyMode copy_mode = ObjectStorageCopyMode::Default, - std::shared_ptr object_creation_client = nullptr); + std::shared_ptr object_creation_client = nullptr, + OnObjectCreationFailure on_object_creation_failure = {}); /// Copies exactly `[src_offset, src_offset + src_size)` of a LARGER source object of size `src_object_size`. /// @@ -83,7 +90,8 @@ void copyS3FileRange( BlobStorageLogWriterPtr blob_storage_log, ThreadPoolCallbackRunnerUnsafe schedule, const CreateReadBuffer & fallback_file_reader, - const std::optional & object_metadata = std::nullopt); + const std::optional & object_metadata = std::nullopt, + OnObjectCreationFailure on_object_creation_failure = {}); /// Copies data from any seekable source to S3. /// The same functionality can be done by using the function copyData() and the class WriteBufferFromS3 @@ -100,7 +108,8 @@ void copyDataToS3File( const S3::S3RequestSettings & settings, BlobStorageLogWriterPtr blob_storage_log, ThreadPoolCallbackRunnerUnsafe schedule, - const std::optional & object_metadata = std::nullopt); + const std::optional & object_metadata = std::nullopt, + OnObjectCreationFailure on_object_creation_failure = {}); } diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 2b4be44f26c57..93fa2372867b1 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -111,7 +111,8 @@ WriteBufferFromS3::WriteBufferFromS3( std::optional object_metadata_, ThreadPoolCallbackRunnerUnsafe schedule_, const WriteSettings & write_settings_, - std::shared_ptr object_creation_client_) + std::shared_ptr object_creation_client_, + OnUploadComplete on_upload_complete_) : WriteBufferFromFileBase(std::min(buf_size_, static_cast(DBMS_DEFAULT_BUFFER_SIZE)), nullptr, 0) , bucket(bucket_) , key(key_) @@ -120,6 +121,7 @@ WriteBufferFromS3::WriteBufferFromS3( , client_ptr(std::move(client_ptr_)) , object_creation_client(object_creation_client_ ? std::move(object_creation_client_) : client_ptr) , object_metadata(std::move(object_metadata_)) + , on_upload_complete(std::move(on_upload_complete_)) , buffer_allocation_policy(createBufferAllocationPolicy(request_settings)) , task_tracker( std::make_unique( @@ -225,21 +227,37 @@ void WriteBufferFromS3::finalizeImpl() chassert(offset() == 0); chassert(hidden_size == 0); - task_tracker->waitAll(); + try + { + task_tracker->waitAll(); - span.addAttributeIfNotZero("clickhouse.multipart_upload_parts", multipart_tags.size()); + span.addAttributeIfNotZero("clickhouse.multipart_upload_parts", multipart_tags.size()); - if (request_settings[S3RequestSetting::check_objects_after_upload]) + if (request_settings[S3RequestSetting::check_objects_after_upload]) + { + S3::checkObjectExists(*client_ptr, bucket, key, {}, "Immediately after upload"); + + size_t actual_size = S3::getObjectSize(*client_ptr, bucket, key, {}); + if (actual_size != total_size) + throw Exception( + ErrorCodes::S3_ERROR, + "Object {} from bucket {} has unexpected size {} after upload, expected size {}, it's a bug in S3 or S3 API.", + key, bucket, actual_size, total_size); + } + } + catch (...) { - S3::checkObjectExists(*client_ptr, bucket, key, {}, "Immediately after upload"); - - size_t actual_size = S3::getObjectSize(*client_ptr, bucket, key, {}); - if (actual_size != total_size) - throw Exception( - ErrorCodes::S3_ERROR, - "Object {} from bucket {} has unexpected size {} after upload, expected size {}, it's a bug in S3 or S3 API.", - key, bucket, actual_size, total_size); + /// `PutObject` or `CompleteMultipartUpload` may have committed the object even when this + /// request failed (a lost response is reported as an error). Drop the cached identity so the + /// next read does not send the previous ETag. A part upload that failed before either request + /// leaves `object_creation_request_sent` false: the visible object is unchanged. + if (object_creation_request_sent && on_upload_complete) + on_upload_complete(); + throw; } + + if (on_upload_complete) + on_upload_complete(); } void WriteBufferFromS3::cancelImpl() noexcept @@ -705,6 +723,9 @@ bool WriteBufferFromS3::completeMultipartUpload() ProfileEvents::increment(ProfileEvents::DiskS3CompleteMultipartUpload); Stopwatch watch; + /// Set before the call. A failure after S3 has committed the upload must still be visible to + /// `finalizeImpl`, which drops the cached identity only once this request has been sent. + object_creation_request_sent = true; auto outcome = object_creation_client->CompleteMultipartUpload(req); auto elapsed = watch.elapsedMicroseconds(); @@ -811,6 +832,8 @@ void WriteBufferFromS3::makeSinglepartUpload(WriteBufferFromS3::PartData && data CurrentThread::WriteThrottlingScope write_throttling_scope(write_settings.remote_throttler); Stopwatch watch; + /// Set before the call. A lost `PutObject` response can still have replaced the object. + object_creation_request_sent = true; auto outcome = object_creation_client->PutObject(request); auto elapsed = watch.elapsedMicroseconds(); diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h index 5c54b4d50717e..30a551b9db6bc 100644 --- a/src/IO/WriteBufferFromS3.h +++ b/src/IO/WriteBufferFromS3.h @@ -16,7 +16,9 @@ #include #include +#include #include +#include #include #include @@ -35,6 +37,9 @@ class TaskTracker; class WriteBufferFromS3 final : public WriteBufferFromFileBase { public: + /// Called after the upload has replaced the object, including when the post-upload size check fails. + using OnUploadComplete = std::function; + WriteBufferFromS3( std::shared_ptr client_ptr_, const String & bucket_, @@ -45,7 +50,8 @@ class WriteBufferFromS3 final : public WriteBufferFromFileBase std::optional object_metadata_ = std::nullopt, ThreadPoolCallbackRunnerUnsafe schedule_ = {}, const WriteSettings & write_settings_ = {}, - std::shared_ptr object_creation_client_ = nullptr); + std::shared_ptr object_creation_client_ = nullptr, + OnUploadComplete on_upload_complete_ = {}); ~WriteBufferFromS3() override; void nextImpl() override; @@ -100,6 +106,10 @@ class WriteBufferFromS3 final : public WriteBufferFromFileBase /// by getResultObjectETag() after finalize(). Written by the upload worker, read after the /// finalize barrier (happens-before), so no extra synchronization is needed. std::optional object_etag; + OnUploadComplete on_upload_complete; + /// Set by the upload worker immediately before `PutObject` or `CompleteMultipartUpload`. A lost + /// response can still have committed the object. Read after `waitAll` (happens-before). + bool object_creation_request_sent = false; LoggerPtr log = getLogger("WriteBufferFromS3"); LogSeriesLimiterPtr limited_log = std::make_shared(log, 1, 5); diff --git a/src/IO/tests/gtest_writebuffer_s3.cpp b/src/IO/tests/gtest_writebuffer_s3.cpp index 686a8b4d61626..da386b8e4ca31 100644 --- a/src/IO/tests/gtest_writebuffer_s3.cpp +++ b/src/IO/tests/gtest_writebuffer_s3.cpp @@ -811,7 +811,10 @@ class WBS3Test : public ::testing::Test return *async_policy; } - std::unique_ptr getWriteBuffer(String file_name = "file", const WriteSettings & write_settings = {}) + std::unique_ptr getWriteBuffer( + String file_name = "file", + const WriteSettings & write_settings = {}, + WriteBufferFromS3::OnUploadComplete on_upload_complete = {}) { S3::S3RequestSettings request_settings; request_settings.updateFromSettings(settings, /* if_changed */true, /* validate_settings */false); @@ -829,7 +832,9 @@ class WBS3Test : public ::testing::Test nullptr, std::nullopt, getAsyncPolicy().getScheduler(), - write_settings); + write_settings, + nullptr, + std::move(on_upload_complete)); } void setInjectionModel(std::shared_ptr injections_) @@ -1067,6 +1072,35 @@ TEST_P(SyncAsync, ExceptionOnCreateMPU) { } +/// `CompleteMultipartUpload` can commit the object and still report an error (a lost response). +/// `finalize` must run the upload callback before propagating that error, so the identity cache +/// does not keep the previous ETag. A part-upload failure never sends the completing request and +/// must not run the callback. +TEST_P(SyncAsync, ExceptionOnCompleteMPURunsUploadCallback) { + setInjectionModel(std::make_shared()); + + getSettings()[Setting::s3_max_single_part_upload_size] = 0; // no single part + getSettings()[Setting::s3_min_upload_part_size] = 1; // small parts are ok + + size_t callbacks = 0; + EXPECT_THROW({ + try { + auto buffer = getWriteBuffer("exception_on_complete_mpu_callback", {}, [&]() { ++callbacks; }); + buffer->write('A'); + + getAsyncPolicy().setAutoExecute(true); + buffer->finalize(); + } + catch(const DB::Exception & e) + { + ASSERT_EQ(ErrorCodes::S3_ERROR, e.code()); + EXPECT_THAT(e.what(), testing::HasSubstr("CompleteMPUFailIngection")); + throw; + } + }, DB::S3Exception); + EXPECT_EQ(callbacks, 1u); +} + TEST_P(SyncAsync, ExceptionOnCompleteMPU) { setInjectionModel(std::make_shared()); @@ -1161,6 +1195,97 @@ TEST_F(WBS3Test, CopyDataToS3FileRetriesInvalidPart) { EXPECT_EQ(bStore.objects["copy_data_invalid_part_retry"].size(), payload.size()); } +/// `CompleteMultipartUpload` can commit the object and still report an error. `copyDataToS3File` must +/// run the failure callback before propagating that error, so a backup copy drops the cached identity. +/// An `UploadPart` failure never sends the completing request and must not run the callback. +TEST_F(WBS3Test, CopyDataToS3FileRunsCallbackWhenCompleteFails) +{ + setInjectionModel(std::make_shared()); + + getSettings()[Setting::s3_max_single_part_upload_size] = 0; + getSettings()[Setting::s3_min_upload_part_size] = 1; + getSettings()[Setting::s3_check_objects_after_upload] = false; + + S3::S3RequestSettings request_settings; + request_settings.updateFromSettings(settings, /* if_changed */ true, /* validate_settings */ false); + + const String payload = "copy_complete_fail_payload"; + auto create_read_buffer = [&]() -> std::unique_ptr + { + return std::make_unique(payload); + }; + + size_t callbacks = 0; + EXPECT_THROW({ + try + { + copyDataToS3File( + create_read_buffer, + /* offset= */ 0, + /* size= */ payload.size(), + client, + bucket, + "copy_data_complete_fail_callback", + request_settings, + /* blob_storage_log= */ nullptr, + /* schedule= */ {}, + /* object_metadata= */ std::nullopt, + [&]() { ++callbacks; }); + } + catch (const DB::Exception & e) + { + ASSERT_EQ(ErrorCodes::S3_ERROR, e.code()); + EXPECT_THAT(e.what(), testing::HasSubstr("CompleteMPUFailIngection")); + throw; + } + }, DB::S3Exception); + EXPECT_EQ(callbacks, 1u); +} + +TEST_F(WBS3Test, CopyDataToS3FileSkipsCallbackWhenPartUploadFails) +{ + setInjectionModel(std::make_shared()); + + getSettings()[Setting::s3_max_single_part_upload_size] = 0; + getSettings()[Setting::s3_min_upload_part_size] = 1; + getSettings()[Setting::s3_check_objects_after_upload] = false; + + S3::S3RequestSettings request_settings; + request_settings.updateFromSettings(settings, /* if_changed */ true, /* validate_settings */ false); + + const String payload = "copy_part_fail_payload"; + auto create_read_buffer = [&]() -> std::unique_ptr + { + return std::make_unique(payload); + }; + + size_t callbacks = 0; + EXPECT_THROW({ + try + { + copyDataToS3File( + create_read_buffer, + /* offset= */ 0, + /* size= */ payload.size(), + client, + bucket, + "copy_data_part_fail_callback", + request_settings, + /* blob_storage_log= */ nullptr, + /* schedule= */ {}, + /* object_metadata= */ std::nullopt, + [&]() { ++callbacks; }); + } + catch (const DB::Exception & e) + { + ASSERT_EQ(ErrorCodes::S3_ERROR, e.code()); + EXPECT_THAT(e.what(), testing::HasSubstr("UploadPartFailIngection")); + throw; + } + }, DB::S3Exception); + EXPECT_EQ(callbacks, 0u); +} + /// copyS3File routing between whole-object CopyObject and ranged UploadPartCopy. A small copy would take /// CopyObject, which carries no byte range and copies the ENTIRE source; a partial-range copy must therefore /// force UploadPartCopy, which sets a CopySourceRange per part -- but only when S3 would accept the source as diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 2766158e3b153..a77bebe2f16cb 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -47,6 +47,7 @@ #include #include #include +#include #include #include #include @@ -619,6 +620,7 @@ struct ContextSharedPart : boost::noncopyable mutable ResourceManagerPtr resource_manager; mutable UncompressedCachePtr uncompressed_cache TSA_GUARDED_BY(mutex); /// The cache of decompressed blocks. mutable MarkCachePtr mark_cache TSA_GUARDED_BY(mutex); /// Cache of marks in compressed files. + mutable ObjectStorageIdentityCachePtr object_storage_identity_cache TSA_GUARDED_BY(mutex); /// Cache of object-storage identity (size/etag/Last-Modified) to avoid per-open HEADs. mutable UniqueKeyIndexCachePtr unique_key_index_cache TSA_GUARDED_BY(mutex); /// RocksDB-compatible block cache over CacheBase for the UNIQUE KEY index (nullptr when RocksDB unavailable or disabled). mutable DeleteBitmapCachePtr delete_bitmap_cache TSA_GUARDED_BY(mutex); /// UNIQUE KEY per-part delete-bitmap cache. mutable PrimaryIndexCachePtr primary_index_cache TSA_GUARDED_BY(mutex); @@ -4694,6 +4696,59 @@ void Context::clearMarkCache() const JemallocCacheArena::purge(); } +void Context::setObjectStorageIdentityCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio) +{ + std::lock_guard lock(shared->mutex); + + if (shared->object_storage_identity_cache) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Object storage identity cache has been already created."); + + /// A zero size disables the cache: leave the pointer unset so callers fall back to uncached reads. + /// `updateObjectStorageIdentityCacheConfiguration` creates it later if a reload sets a non-zero size. + if (max_size_in_bytes == 0) + return; + + shared->object_storage_identity_cache = std::make_shared(cache_policy, max_size_in_bytes, size_ratio); +} + +void Context::updateObjectStorageIdentityCacheConfiguration(const Poco::Util::AbstractConfiguration & config, size_t max_cache_size) +{ + std::lock_guard lock(shared->mutex); + + size_t size = config.getUInt64("object_storage_identity_cache_size", 67108864); + if (size > max_cache_size) + size = max_cache_size; + + /// Startup with size 0 leaves the pointer unset. A reload that raises the size has to create the + /// cache here: returning would keep every read uncached until the next process start. + if (!shared->object_storage_identity_cache) + { + if (size == 0) + return; + + const String policy = config.getString("object_storage_identity_cache_policy", "SLRU"); + const double size_ratio = config.getDouble("object_storage_identity_cache_size_ratio", 0.5); + shared->object_storage_identity_cache = std::make_shared(policy, size, size_ratio); + return; + } + + shared->object_storage_identity_cache->setMaxSizeInBytes(size); +} + +std::shared_ptr Context::getObjectStorageIdentityCache() const +{ + SharedLockGuard lock(shared->mutex); + return shared->object_storage_identity_cache; +} + +void Context::clearObjectStorageIdentityCache() const +{ + ObjectStorageIdentityCachePtr cache = getObjectStorageIdentityCache(); + + if (cache) + cache->clear(); +} + void Context::setUniqueKeyIndexCache( [[maybe_unused]] const String & cache_policy, [[maybe_unused]] size_t max_cache_size_in_bytes, diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 93afbfed42416..3be8811d2359a 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -104,6 +104,7 @@ class RefreshSet; class Cluster; class Compiler; class MarkCache; +class ObjectStorageIdentityCache; class UniqueKeyIndexCache; class DeleteBitmapCache; class PrimaryIndexCache; @@ -1570,6 +1571,13 @@ class Context: public ContextData, public std::enable_shared_from_this void clearMarkCache() const; ThreadPool & getLoadMarksThreadpool() const; + /// Object-storage identity cache (per-object size/ETag/Last-Modified) to avoid a HEAD + /// on every object open. A max_size_in_bytes of 0 leaves the cache unset (disabled). + void setObjectStorageIdentityCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio); + void updateObjectStorageIdentityCacheConfiguration(const Poco::Util::AbstractConfiguration & config, size_t max_cache_size); + std::shared_ptr getObjectStorageIdentityCache() const; + void clearObjectStorageIdentityCache() const; + /// UNIQUE KEY index cache: ClickHouse-side `CacheBase` adapter /// over the RocksDB block cache used by SST-backed UNIQUE KEY indexes. void setUniqueKeyIndexCache(const String & cache_policy, size_t max_cache_size_in_bytes, double size_ratio); diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index be76b6c659362..c74b192881060 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -515,6 +515,10 @@ BlockIO InterpreterSystemQuery::execute() #else throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "The server was compiled without the support for Parquet"); #endif + case Type::CLEAR_OBJECT_STORAGE_IDENTITY_CACHE: + getContext()->checkAccess(AccessType::SYSTEM_DROP_OBJECT_STORAGE_IDENTITY_CACHE); + system_context->clearObjectStorageIdentityCache(); + break; case Type::CLEAR_POINT_IN_POLYGON_CACHE: getContext()->checkAccess(AccessType::SYSTEM_DROP_POINT_IN_POLYGON_CACHE); clearPointInPolygonCache(); @@ -3298,6 +3302,9 @@ AccessRightsElements InterpreterSystemQuery::getRequiredAccessForDDLOnCluster() case Type::CLEAR_PARQUET_METADATA_CACHE: required_access.emplace_back(AccessType::SYSTEM_DROP_PARQUET_METADATA_CACHE); break; + case Type::CLEAR_OBJECT_STORAGE_IDENTITY_CACHE: + required_access.emplace_back(AccessType::SYSTEM_DROP_OBJECT_STORAGE_IDENTITY_CACHE); + break; case Type::CLEAR_POINT_IN_POLYGON_CACHE: required_access.emplace_back(AccessType::SYSTEM_DROP_POINT_IN_POLYGON_CACHE); break; diff --git a/src/Parsers/ASTSystemQuery.cpp b/src/Parsers/ASTSystemQuery.cpp index c5da6d8dffdfc..a3f011d06c7e0 100644 --- a/src/Parsers/ASTSystemQuery.cpp +++ b/src/Parsers/ASTSystemQuery.cpp @@ -673,6 +673,7 @@ void ASTSystemQuery::formatImpl(WriteBuffer & ostr, const FormatSettings & setti case Type::CLEAR_ICEBERG_METADATA_CACHE: case Type::CLEAR_PAIMON_METADATA_CACHE: case Type::CLEAR_PARQUET_METADATA_CACHE: + case Type::CLEAR_OBJECT_STORAGE_IDENTITY_CACHE: case Type::CLEAR_POINT_IN_POLYGON_CACHE: case Type::CLEAR_AVRO_SCHEMA_CACHE: case Type::DROP_OBJECT_STORAGE_LIST_OBJECTS_CACHE: diff --git a/src/Parsers/ASTSystemQuery.h b/src/Parsers/ASTSystemQuery.h index d6cb01ba5f909..d18eac0d789b1 100644 --- a/src/Parsers/ASTSystemQuery.h +++ b/src/Parsers/ASTSystemQuery.h @@ -49,6 +49,7 @@ class ASTSystemQuery : public IAST, public ASTQueryWithOnCluster CLEAR_ICEBERG_METADATA_CACHE, CLEAR_PAIMON_METADATA_CACHE, CLEAR_PARQUET_METADATA_CACHE, + CLEAR_OBJECT_STORAGE_IDENTITY_CACHE, CLEAR_POINT_IN_POLYGON_CACHE, CLEAR_PUFFIN_FILES_CACHE, CLEAR_FILESYSTEM_CACHE, diff --git a/src/Parsers/ParserSystemQuery.cpp b/src/Parsers/ParserSystemQuery.cpp index c6864c2c9c561..2486db703ff91 100644 --- a/src/Parsers/ParserSystemQuery.cpp +++ b/src/Parsers/ParserSystemQuery.cpp @@ -322,6 +322,7 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & {"DROP ICEBERG METADATA CACHE", Type::CLEAR_ICEBERG_METADATA_CACHE}, {"DROP PAIMON METADATA CACHE", Type::CLEAR_PAIMON_METADATA_CACHE}, {"DROP PARQUET METADATA CACHE", Type::CLEAR_PARQUET_METADATA_CACHE}, + {"DROP OBJECT STORAGE IDENTITY CACHE", Type::CLEAR_OBJECT_STORAGE_IDENTITY_CACHE}, {"DROP POINT IN POLYGON CACHE", Type::CLEAR_POINT_IN_POLYGON_CACHE}, {"DROP PUFFIN FILES CACHE", Type::CLEAR_PUFFIN_FILES_CACHE}, {"DROP PUFFIN_FILES_CACHE", Type::CLEAR_PUFFIN_FILES_CACHE}, diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.cpp index 0f6a1b33f66bf..14c3050cee5e0 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.cpp @@ -100,6 +100,10 @@ IcebergDataObjectInfo::IcebergDataObjectInfo( , resolved_storage(std::move(resolved_storage_)) { chassert(resolved_key_.empty() == (resolved_storage == nullptr)); + /// Object identity (size / etag / Last-Modified) is intentionally not pre-populated from + /// the manifest. S3ObjectStorage::getObjectMetadata loads it once and keeps it in + /// ObjectStorageIdentityCache, so Iceberg data files get the real ETag used by the filesystem, + /// page, and Parquet metadata caches. Pre-populating metadata here would skip that path. } IcebergDataObjectInfo::IcebergDataObjectInfo(const RelativePathWithMetadata & path_) diff --git a/src/Storages/ObjectStorage/ReadBufferIterator.cpp b/src/Storages/ObjectStorage/ReadBufferIterator.cpp index 8874a4be95837..1886612ebf774 100644 --- a/src/Storages/ObjectStorage/ReadBufferIterator.cpp +++ b/src/Storages/ObjectStorage/ReadBufferIterator.cpp @@ -93,8 +93,9 @@ std::optional ReadBufferIterator::tryGetColumnsFromCache( /// An unknown modification time (e.g. a web object whose HTTP response has no `Last-Modified` /// header) must be reported as unavailable, not as the epoch: the schema cache reuses a cached /// entry whenever the modification time is older than the registration time, so a fake epoch-`0` - /// would keep a stale schema valid forever. Returning `nullopt` makes the cache skip the entry. - if (!metadata || !metadata->is_last_modified_known) + /// would keep a stale schema valid forever. A time reused from the identity cache has the same + /// effect after an external replace. Returning `nullopt` makes the cache skip the entry. + if (!metadata || !metadata->lastModifiedConfirmsUnchanged()) return std::nullopt; return std::optional(metadata->last_modified.epochTime()); }; @@ -257,6 +258,12 @@ ReadBufferIterator::Data ReadBufferIterator::next() prev_read_keys_size = read_keys.size(); } + if (query_settings.skip_empty_files && current_object_info->getObjectMetadata() + && current_object_info->getObjectMetadata()->is_from_identity_cache + && current_object_info->getObjectMetadata()->size_bytes == 0 + && current_object_info->getObjectMetadata()->is_size_known) + revalidateIdentityCachedObjectMetadata(*current_object_info, object_storage); + if (query_settings.skip_empty_files && current_object_info->getObjectMetadata() && current_object_info->getObjectMetadata()->size_bytes == 0 && current_object_info->getObjectMetadata()->is_size_known) diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index e73a1383c6007..bf72ca5aff661 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -1163,19 +1163,38 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade } } - if (query_settings.skip_empty_files && object_info->getObjectMetadata()->size_bytes == 0 + /// The cached size is the previous object's. Re-HEAD before treating it as empty, or a + /// replaced object is skipped and never read. `s3_skip_empty_files` defaults to true. + const auto cached_metadata = object_info->getObjectMetadata(); + if (cached_metadata + && cached_metadata->is_from_identity_cache + && query_settings.skip_empty_files + && cached_metadata->is_size_known + && cached_metadata->size_bytes == 0) + revalidateIdentityCachedObjectMetadata(*object_info, object_storage); + + if (query_settings.skip_empty_files && object_info->getObjectMetadata() + && object_info->getObjectMetadata()->size_bytes == 0 && object_info->getObjectMetadata()->is_size_known) continue; + bool skip_object = false; if (query_condition_cache && !object_info->file_bucket_info) { - const auto query_condition_cache_key = makeQueryConditionCacheKey(*object_info, configuration->isDataLakeConfiguration()); - std::optional matching_marks; - if (query_condition_cache_key) - matching_marks = query_condition_cache->read( - storage_id.uuid, *query_condition_cache_key, *format_filter_info->condition_hash); - if (matching_marks.has_value()) + /// A hit that drops every row group never GETs. The key contains the ETag, and an + /// identity-cache hit still has the previous one, so confirm it before dropping the file. + /// Data-lake keys do not contain the ETag. One confirmation is enough: the HEAD clears + /// `is_from_identity_cache`. + for (size_t attempt = 0; attempt < 2; ++attempt) { + const auto query_condition_cache_key = makeQueryConditionCacheKey(*object_info, configuration->isDataLakeConfiguration()); + std::optional matching_marks; + if (query_condition_cache_key) + matching_marks = query_condition_cache->read( + storage_id.uuid, *query_condition_cache_key, *format_filter_info->condition_hash); + if (!matching_marks.has_value()) + break; + const auto & marks = *matching_marks; size_t total_row_groups = marks.size(); std::vector matching_row_groups; @@ -1191,20 +1210,47 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade format_filter_info->filter_actions_dag->dumpNames(), object_info->getFileName()); - if (matching_row_groups.empty()) - continue; + bool drops_object = matching_row_groups.empty(); + FileBucketInfoPtr filtered; + if (!drops_object) + { + auto file_bucket_info = FormatFactory::instance().getFileBucketInfo( + object_info->getFileFormat().value_or(configuration->getFormat())); + if (file_bucket_info) + { + filtered = file_bucket_info->filterByMatchingRowGroups(matching_row_groups); + if (!filtered) + drops_object = true; + } + } - auto file_bucket_info = FormatFactory::instance().getFileBucketInfo( - object_info->getFileFormat().value_or(configuration->getFormat())); - if (file_bucket_info) + const auto metadata_now = object_info->getObjectMetadata(); + if (drops_object && attempt == 0 && metadata_now && metadata_now->is_from_identity_cache + && !configuration->isDataLakeConfiguration()) { - auto filtered = file_bucket_info->filterByMatchingRowGroups(matching_row_groups); - if (!filtered) - continue; - object_info->file_bucket_info = std::move(filtered); + revalidateIdentityCachedObjectMetadata(*object_info, object_storage); + + /// The empty-file check above already ran, against the previous size. A replaced + /// object can be empty, and reading it throws for formats such as Parquet. + /// `s3_skip_empty_files` defaults to true. + const auto refreshed = object_info->getObjectMetadata(); + if (query_settings.skip_empty_files && refreshed + && refreshed->is_size_known && refreshed->size_bytes == 0) + { + skip_object = true; + break; + } + continue; } + + skip_object = drops_object; + if (filtered) + object_info->file_bucket_info = std::move(filtered); + break; } } + if (skip_object) + continue; break; } @@ -1237,9 +1283,10 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade { const auto metadata = object_info->getObjectMetadata(); /// An unknown modification time (e.g. a web object without a `Last-Modified` header) must not be - /// reported as the epoch, otherwise the stale cached row count would always look valid. Reporting - /// it as unavailable makes the count cache re-read the file instead. - if (!metadata || !metadata->is_last_modified_known) + /// reported as the epoch, otherwise the stale cached row count would always look valid. A time + /// reused from the identity cache is also not proof the object is unchanged. Reporting either + /// as unavailable makes the count cache re-read the file instead. + if (!metadata || !metadata->lastModifiedConfirmsUnchanged()) return std::nullopt; return std::optional(metadata->last_modified.epochTime()); }; @@ -1456,8 +1503,21 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade { ProfileEvents::increment(ProfileEvents::ObjectStorageReadObjects); compression_method = chooseCompressionMethod(object_info->getFileName(), configuration->compression_method); + /// A Parquet `count()` reads the cached footer and does not GET, so `If-Match` never + /// rejects a replaced object. Re-HEAD before that ETag keys the parquet metadata cache. + /// A read that does GET still checks the ETag there and does not pay this HEAD. + const bool revalidate_identity_for_parquet_metadata_cache = + effective_need_only_count + && context_->getSettingsRef()[Setting::use_parquet_metadata_cache] + && Poco::toLower(format_name) == "parquet"; read_buf = createReadBuffer( - object_info->relative_path_with_metadata, object_info->getResolvedStorage(object_storage), context_, log, std::nullopt, !headers_requested); + object_info->relative_path_with_metadata, + object_info->getResolvedStorage(object_storage), + context_, + log, + std::nullopt, + !headers_requested, + revalidate_identity_for_parquet_metadata_cache); } Block initial_header = read_from_format_info.format_header; @@ -1921,13 +1981,27 @@ std::future StorageObjectStorageSource return create_reader_scheduler([=, this] { return createReader(); }, Priority{}); } +void revalidateIdentityCachedObjectMetadata(ObjectInfo & object_info, const ObjectStoragePtr & object_storage) +{ + const auto metadata = object_info.getObjectMetadata(); + if (!metadata || !metadata->is_from_identity_cache) + return; + + auto metadata_object = object_info.relative_path_with_metadata; + metadata_object.relative_path = object_info.getPathOrPathToArchiveIfArchive(); + object_info.setObjectMetadata( + object_info.getResolvedStorage(object_storage)->getObjectMetadataIgnoringIdentityCache( + metadata_object, /*with_tags=*/ false)); +} + std::unique_ptr createReadBuffer( RelativePathWithMetadata & object_info, const ObjectStoragePtr & object_storage, const ContextPtr & context_, const LoggerPtr & log, const std::optional & read_settings, - bool allow_page_cache) + bool allow_page_cache, + bool revalidate_identity_for_parquet_metadata_cache) { const auto & settings = context_->getSettingsRef(); const auto & effective_read_settings = read_settings.has_value() ? read_settings.value() : context_->getReadSettings(); @@ -1983,6 +2057,24 @@ std::unique_ptr createReadBuffer( object_info.metadata = object_storage->getObjectMetadata(object_info, /*with_tags=*/ false); } + /// A filesystem, page, or parquet-metadata cache hit never issues the GET, so it cannot notice + /// that the object was replaced. Re-read the identity before using its ETag as the cache key. + /// An unchanged object keeps the same key. A replaced object gets a new key and is read again. + /// The parquet flag is only a footer-only count: other Parquet reads still check the ETag on GET. + /// + /// The same re-read is required when this query does not check the ETag at all. `getObjectMetadata` + /// skips the hit when the query context carries `s3_validate_etag_on_read = 0`, but a hit taken + /// without that context still has the previous size, and a small-file prefetch stops at that size. + const bool identity_from_cache = object_info.metadata && object_info.metadata->is_from_identity_cache; + const bool revalidate_for_content_cache = identity_from_cache + && (use_filesystem_cache || use_page_cache || revalidate_identity_for_parquet_metadata_cache) + && object_info.metadata->is_size_known + && object_info.metadata->isEtagUsableAsCacheKey(); + if (revalidate_for_content_cache || (identity_from_cache && !validate_etag_on_read)) + { + object_info.metadata = object_storage->getObjectMetadataIgnoringIdentityCache(object_info, /*with_tags=*/ false); + } + if (use_page_cache && !object_info.metadata->isEtagUsableAsCacheKey()) { LOG_WARNING(log, "Cannot use page cache, etag is missing or not a strong content identifier"); diff --git a/src/Storages/ObjectStorage/Utils.h b/src/Storages/ObjectStorage/Utils.h index 096b401f9852e..fe165198c54a5 100644 --- a/src/Storages/ObjectStorage/Utils.h +++ b/src/Storages/ObjectStorage/Utils.h @@ -31,13 +31,18 @@ void validateSupportedColumns( /// An empty column name has no identifier to render it with, so it cannot survive analysis. void validateLakeSchemaColumnNames(const NamesAndTypesList & schema, std::string_view lake_name); +/// Replace identity-cache metadata with a HEAD. A hit still has the previous size and ETag, and a caller +/// that then skips the read never issues the GET that would notice a replace. +void revalidateIdentityCachedObjectMetadata(ObjectInfo & object_info, const ObjectStoragePtr & object_storage); + std::unique_ptr createReadBuffer( RelativePathWithMetadata & object_info, const ObjectStoragePtr & object_storage, const ContextPtr & context_, const LoggerPtr & log, const std::optional & read_settings = std::nullopt, - bool allow_page_cache = true); + bool allow_page_cache = true, + bool revalidate_identity_for_parquet_metadata_cache = false); ASTs::iterator getFirstKeyValueArgument(ASTs & args); std::unordered_map parseKeyValueArguments(const ASTs & function_args, ContextPtr context); diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueuePostProcessor.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueuePostProcessor.cpp index c520d97da2bb0..4bb5ba146c246 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueuePostProcessor.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueuePostProcessor.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -363,6 +364,17 @@ void ObjectStorageQueuePostProcessor::moveS3Objects(const StoredObjects & object object_from.remote_path); auto object_to = applyMovePrefixIfPresent(object_from, move_prefix, settings.after_processing_move_preserve_path); + /// This copy does not go through `S3ObjectStorage`, so nothing else drops the + /// destination's cached identity. A lost `CopyObject` or `CompleteMultipartUpload` + /// response can still have replaced the object, so the callback covers that throw. + const String endpoint = dst_uri.endpoint; + const String bucket = dst_uri.bucket; + const String dest_key = object_to.remote_path; + auto invalidate_destination = [endpoint, bucket, dest_key]() + { + invalidateObjectStorageIdentityCache(endpoint, bucket, dest_key); + }; + LOG_INFO(log, "Copying {} ({} Bytes) to bucket {}", object_from.remote_path, object_size, dst_uri.bucket); copyS3File( src_client, @@ -371,15 +383,20 @@ void ObjectStorageQueuePostProcessor::moveS3Objects(const StoredObjects & object /*src_size=*/ object_size, /*dest_s3_client=*/ dst_client, /*dest_bucket=*/ dst_uri.bucket, - /*dest_key=*/ object_to.remote_path, + /*dest_key=*/ dest_key, /*settings=*/ s3_settings->request_settings, /*read_settings=*/ read_settings_to_use, BlobStorageLogWriter::create(object_storage->getDiskName()), scheduler, - /*fallback_file_reader=*/ [&]{ + /*fallback_file_reader=*/ [&] + { return s3_storage->readObject(object_from, read_settings_to_use); - } - /*object_metadata=*/); + }, + /*object_metadata=*/ std::nullopt, + ObjectStorageCopyMode::Default, + /*object_creation_client=*/ nullptr, + invalidate_destination); + invalidate_destination(); LOG_INFO(log, "Removing object {}", object_from.remote_path); object_storage->removeObjectIfExists(object_from); diff --git a/tests/integration/test_database_iceberg/test.py b/tests/integration/test_database_iceberg/test.py index 98d41bad325a2..5011ddbb9c78d 100644 --- a/tests/integration/test_database_iceberg/test.py +++ b/tests/integration/test_database_iceberg/test.py @@ -1137,6 +1137,9 @@ def test_select_manifest_without_sequence_number(started_cluster, rewrite_manife ["sequence_number", "min_sequence_number", "content"], ) + # The rewrite goes straight to MinIO. The identity cache still holds the old ETag, and the next + # GET would send it as If-Match and fail. + node.query("SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE") node.query("SYSTEM DROP ICEBERG METADATA CACHE") assert num_rows == int(node.query(f"SELECT count() FROM (SELECT * FROM {table_ref})")) @@ -1183,6 +1186,9 @@ def test_select_manifest_without_snapshot_id(started_cluster, rewrite_manifest_l ["sequence_number", "min_sequence_number", "content"], ) + # The rewrite goes straight to MinIO. The identity cache still holds the old ETag, and the next + # GET would send it as If-Match and fail. + node.query("SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE") node.query("SYSTEM DROP ICEBERG METADATA CACHE") assert num_rows == int(node.query(f"SELECT count() FROM (SELECT * FROM {table_ref})")) diff --git a/tests/integration/test_storage_gcp_auth/test.py b/tests/integration/test_storage_gcp_auth/test.py index f898ce417ea70..f57ffe9be7453 100644 --- a/tests/integration/test_storage_gcp_auth/test.py +++ b/tests/integration/test_storage_gcp_auth/test.py @@ -131,7 +131,9 @@ def get_num_requests(): assert get_num_requests() == 2 assert node.query("SELECT * FROM s3_table") == "OK\n" - assert get_num_requests() == 4 + # The first read stored this object's identity, so this read does not HEAD. One GET fetches one + # token. A cold read would HEAD and GET and add two. + assert get_num_requests() == 3 assert ( node.query( "SELECT * FROM s3(gcs_conn, filename='test.txt', format='LineAsString')" diff --git a/tests/integration/test_storage_iceberg_no_spark/test_iceberg_inverted_manifest_bounds.py b/tests/integration/test_storage_iceberg_no_spark/test_iceberg_inverted_manifest_bounds.py index e752d00a60520..e86a7072f4107 100644 --- a/tests/integration/test_storage_iceberg_no_spark/test_iceberg_inverted_manifest_bounds.py +++ b/tests/integration/test_storage_iceberg_no_spark/test_iceberg_inverted_manifest_bounds.py @@ -348,6 +348,9 @@ def pruned_files(where): # them, while `id` is still well formed in the same manifest entry and must keep pruning. manifests = MinioManifests(cluster.minio_client, cluster.minio_bucket, key_prefix) assert _swap_bounds_in_manifests(manifests, only_field_id=2) > 0 + # `fput_object` replaces the manifest outside ClickHouse. The identity cache still holds the + # previous ETag, and the next GET sends it as `If-Match`. + instance.query("SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE") assert pruned_files(d_expr) == 0 assert pruned_files(id_expr) == 1 @@ -358,5 +361,6 @@ def pruned_files(where): e_expr = "e < toDecimal32(0.5, 1)" assert pruned_files(e_expr) == 1 assert _set_bounds_in_manifests(manifests, 3, b"\x7f\xff\xff\xf0", b"\x7f\xff\xff\xfa") > 0 + instance.query("SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE") assert pruned_files(e_expr) == 0 assert pruned_files(id_expr) == 1 diff --git a/tests/queries/0_stateless/01271_show_privileges.reference b/tests/queries/0_stateless/01271_show_privileges.reference index 30e02d7c98736..9c05710825876 100644 --- a/tests/queries/0_stateless/01271_show_privileges.reference +++ b/tests/queries/0_stateless/01271_show_privileges.reference @@ -137,6 +137,7 @@ SYSTEM DROP ICEBERG METADATA CACHE ['SYSTEM CLEAR ICEBERG_METADATA_CACHE','SYSTE SYSTEM DROP PAIMON METADATA CACHE ['SYSTEM CLEAR PAIMON_METADATA_CACHE','SYSTEM DROP PAIMON_METADATA_CACHE'] GLOBAL SYSTEM DROP CACHE SYSTEM DROP AVRO SCHEMA CACHE ['SYSTEM CLEAR AVRO SCHEMA CACHE','SYSTEM DROP AVRO SCHEMA CACHE','DROP AVRO SCHEMA CACHE'] GLOBAL SYSTEM DROP CACHE SYSTEM DROP PARQUET METADATA CACHE ['SYSTEM DROP PARQUET_METADATA_CACHE'] GLOBAL SYSTEM DROP CACHE +SYSTEM DROP OBJECT STORAGE IDENTITY CACHE ['SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE','SYSTEM DROP OBJECT STORAGE IDENTITY CACHE'] GLOBAL SYSTEM DROP CACHE SYSTEM DROP POINT IN POLYGON CACHE ['SYSTEM CLEAR POINT IN POLYGON CACHE','SYSTEM DROP POINT IN POLYGON CACHE','DROP POINT IN POLYGON CACHE'] GLOBAL SYSTEM DROP CACHE SYSTEM DROP PUFFIN FILES CACHE ['SYSTEM DROP PUFFIN_FILES_CACHE'] GLOBAL SYSTEM DROP CACHE SYSTEM PREWARM PRIMARY INDEX CACHE ['SYSTEM PREWARM PRIMARY INDEX','PREWARM PRIMARY INDEX CACHE','PREWARM PRIMARY INDEX'] GLOBAL SYSTEM DROP CACHE diff --git a/tests/queries/0_stateless/04033_s3_brace_glob_no_extra_head.sql b/tests/queries/0_stateless/04033_s3_brace_glob_no_extra_head.sql index 8698b4ab874c6..68c1f9deba297 100644 --- a/tests/queries/0_stateless/04033_s3_brace_glob_no_extra_head.sql +++ b/tests/queries/0_stateless/04033_s3_brace_glob_no_extra_head.sql @@ -1,5 +1,6 @@ --- Tags: no-fasttest, no-parallel-replicas +-- Tags: no-fasttest, no-parallel, no-parallel-replicas -- Tag no-fasttest: Depends on AWS +-- Tag no-parallel: the identity cache is process-global, so another test can HEAD these keys first -- Tag no-parallel-replicas: test counts exact S3 API calls per query, which differs under parallel replicas -- Verify that reading from S3 with brace-expansion globs like {a,b,c}.tsv @@ -9,6 +10,9 @@ SET max_threads = 1; SET use_hive_partitioning=1; +-- Drop identities stored by earlier tests. A hit does not send HEAD, so the count below would be 0. +SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE; + SELECT * FROM s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3; diff --git a/tests/queries/0_stateless/04836_system_cache_on_cluster_access_types.reference b/tests/queries/0_stateless/04836_system_cache_on_cluster_access_types.reference index 053c744b102ea..dac14b3fa0d69 100644 --- a/tests/queries/0_stateless/04836_system_cache_on_cluster_access_types.reference +++ b/tests/queries/0_stateless/04836_system_cache_on_cluster_access_types.reference @@ -8,6 +8,7 @@ SYSTEM CLEAR ICEBERG METADATA CACHE -> SYSTEM DROP ICEBERG METADATA CACHE SYSTEM CLEAR PAIMON METADATA CACHE -> SYSTEM DROP PAIMON METADATA CACHE SYSTEM CLEAR AVRO SCHEMA CACHE -> SYSTEM DROP AVRO SCHEMA CACHE SYSTEM CLEAR PARQUET METADATA CACHE -> SYSTEM DROP PARQUET METADATA CACHE +SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE -> SYSTEM DROP OBJECT STORAGE IDENTITY CACHE SYSTEM CLEAR POINT IN POLYGON CACHE -> SYSTEM DROP POINT IN POLYGON CACHE SYSTEM CLEAR PRIMARY INDEX CACHE -> SYSTEM DROP PRIMARY INDEX CACHE SYSTEM CLEAR MMAP CACHE -> SYSTEM DROP MMAP CACHE diff --git a/tests/queries/0_stateless/04836_system_cache_on_cluster_access_types.sh b/tests/queries/0_stateless/04836_system_cache_on_cluster_access_types.sh index 2c748c1570684..92b3b540d7b9f 100755 --- a/tests/queries/0_stateless/04836_system_cache_on_cluster_access_types.sh +++ b/tests/queries/0_stateless/04836_system_cache_on_cluster_access_types.sh @@ -95,6 +95,7 @@ SYSTEM CLEAR ICEBERG METADATA CACHE SYSTEM CLEAR PAIMON METADATA CACHE SYSTEM CLEAR AVRO SCHEMA CACHE SYSTEM CLEAR PARQUET METADATA CACHE +SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE SYSTEM CLEAR POINT IN POLYGON CACHE SYSTEM CLEAR PRIMARY INDEX CACHE SYSTEM CLEAR MMAP CACHE diff --git a/tests/queries/0_stateless/05484_object_storage_identity_cache.reference b/tests/queries/0_stateless/05484_object_storage_identity_cache.reference new file mode 100644 index 0000000000000..d86bac9de59ab --- /dev/null +++ b/tests/queries/0_stateless/05484_object_storage_identity_cache.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/05484_object_storage_identity_cache.sh b/tests/queries/0_stateless/05484_object_storage_identity_cache.sh new file mode 100755 index 0000000000000..ad9283993f230 --- /dev/null +++ b/tests/queries/0_stateless/05484_object_storage_identity_cache.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-parallel-replicas +# Tag no-fasttest: reads a file through the mock S3 server at localhost:11111 +# Tag no-parallel: SYSTEM DROP OBJECT STORAGE IDENTITY CACHE clears the process-global cache +# Tag no-parallel-replicas: per-query S3 request counts change when a read is spread across replicas + +# The object-storage identity cache stores size, etag and Last-Modified so a later open of the same +# key does not send HEAD. A cold read misses and issues one HEAD. The next read of that key is a hit +# and issues none. SYSTEM DROP OBJECT STORAGE IDENTITY CACHE removes the entry, so the following read +# misses again. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +URL="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" +READ_SETTINGS="max_threads = 1, use_hive_partitioning = 0, parallel_replicas_for_cluster_engines = 0, enable_filesystem_cache = 0, use_page_cache_for_object_storage = 0, s3_validate_etag_on_read = 0, use_object_storage_list_objects_cache = 0" + +fail() +{ + echo "FAIL: $*" >&2 + exit 1 +} + +# $1 label, $2 extra settings (may be empty), $3 head mode (eq0|ge1|any), $4 hits mode, $5 misses mode +read_and_check() +{ + local label=$1 + local extra=$2 + local head_mode=$3 + local hits_mode=$4 + local misses_mode=$5 + local qid="idcache_${label}_${CLICKHOUSE_DATABASE}" + local data settings row head hits misses lists + + settings="${READ_SETTINGS}" + if [[ -n $extra ]]; then + settings="${settings}, ${extra}" + fi + + data=$(${CLICKHOUSE_CLIENT} --query_id "$qid" -q \ + "SELECT n FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') SETTINGS ${settings} FORMAT TabSeparated") \ + || fail "${label}: select failed" + [[ $data == "42" ]] || fail "${label}: data '${data}'" + + ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "${label}: flush logs failed" + row=$(${CLICKHOUSE_CLIENT} -q " + SELECT + ProfileEvents['S3HeadObject'], + ProfileEvents['ObjectStorageIdentityCacheHits'], + ProfileEvents['ObjectStorageIdentityCacheMisses'], + ProfileEvents['S3ListObjects'] + FROM system.query_log + WHERE event_date >= yesterday() + AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND query_id = '${qid}' + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC + LIMIT 1 + FORMAT TabSeparated") || fail "${label}: query_log failed" + [[ -n $row ]] || fail "${label}: no query_log row" + + IFS=$'\t' read -r head hits misses lists <<< "$row" + check_count "$label" S3HeadObject "$head" "$head_mode" + check_count "$label" Hits "$hits" "$hits_mode" + check_count "$label" Misses "$misses" "$misses_mode" + [[ $lists == "0" ]] || fail "${label}: S3ListObjects=${lists}, expected 0 (row: ${row})" +} + +check_count() +{ + local label=$1 + local name=$2 + local got=$3 + local mode=$4 + case "$mode" in + eq0) [[ $got == "0" ]] || fail "${label}: ${name}=${got}, expected 0" ;; + ge1) [[ $got -ge 1 ]] || fail "${label}: ${name}=${got}, expected >= 1" ;; + any) ;; + *) fail "${label}: bad mode ${mode}" ;; + esac +} + +${CLICKHOUSE_CLIENT} -q " + INSERT INTO FUNCTION s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') + SELECT 42 + SETTINGS s3_truncate_on_insert = 1" || fail "insert failed" + +# Drop whatever the INSERT cached, then prove a cold read, a hit, and that DROP really clears. +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear failed" +read_and_check cold "" ge1 eq0 ge1 +read_and_check warm "" eq0 ge1 eq0 + +${CLICKHOUSE_CLIENT} -q "SYSTEM DROP OBJECT STORAGE IDENTITY CACHE" || fail "drop failed" +read_and_check after_drop "" ge1 eq0 ge1 + +echo OK diff --git a/tests/queries/0_stateless/05485_object_storage_identity_cache_endpoint_key.reference b/tests/queries/0_stateless/05485_object_storage_identity_cache_endpoint_key.reference new file mode 100644 index 0000000000000..d86bac9de59ab --- /dev/null +++ b/tests/queries/0_stateless/05485_object_storage_identity_cache_endpoint_key.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/05485_object_storage_identity_cache_endpoint_key.sh b/tests/queries/0_stateless/05485_object_storage_identity_cache_endpoint_key.sh new file mode 100755 index 0000000000000..80dcb2b2401fb --- /dev/null +++ b/tests/queries/0_stateless/05485_object_storage_identity_cache_endpoint_key.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-parallel-replicas +# Tag no-fasttest: reads a file through the mock S3 server at localhost:11111 +# Tag no-parallel: SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE clears the process-global cache +# Tag no-parallel-replicas: per-query S3 request counts change when a read is spread across replicas + +# The identity cache key includes the endpoint. localhost and 127.0.0.1 reach the same SeaweedFS +# bucket and key, but they are different endpoint strings, so a hit on one must not answer the other. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +URL_LOCALHOST="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" +URL_LOOPBACK="http://127.0.0.1:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" +READ_SETTINGS="max_threads = 1, use_hive_partitioning = 0, parallel_replicas_for_cluster_engines = 0, enable_filesystem_cache = 0, use_page_cache_for_object_storage = 0, s3_validate_etag_on_read = 0, use_object_storage_list_objects_cache = 0" + +fail() +{ + echo "FAIL: $*" >&2 + exit 1 +} + +# $1 label, $2 url, $3 head mode (eq0|ge1|any), $4 hits mode, $5 misses mode +read_and_check() +{ + local label=$1 + local url=$2 + local head_mode=$3 + local hits_mode=$4 + local misses_mode=$5 + local qid="idcache_ep_${label}_${CLICKHOUSE_DATABASE}" + local data row head hits misses lists + + data=$(${CLICKHOUSE_CLIENT} --query_id "$qid" -q \ + "SELECT n FROM s3('${url}', 'test', 'testtest', 'TSV', 'n UInt64') SETTINGS ${READ_SETTINGS} FORMAT TabSeparated") \ + || fail "${label}: select failed" + [[ $data == "42" ]] || fail "${label}: data '${data}'" + + ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "${label}: flush logs failed" + row=$(${CLICKHOUSE_CLIENT} -q " + SELECT + ProfileEvents['S3HeadObject'], + ProfileEvents['ObjectStorageIdentityCacheHits'], + ProfileEvents['ObjectStorageIdentityCacheMisses'], + ProfileEvents['S3ListObjects'] + FROM system.query_log + WHERE event_date >= yesterday() + AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND query_id = '${qid}' + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC + LIMIT 1 + FORMAT TabSeparated") || fail "${label}: query_log failed" + [[ -n $row ]] || fail "${label}: no query_log row" + + IFS=$'\t' read -r head hits misses lists <<< "$row" + check_count "$label" S3HeadObject "$head" "$head_mode" + check_count "$label" Hits "$hits" "$hits_mode" + check_count "$label" Misses "$misses" "$misses_mode" + [[ $lists == "0" ]] || fail "${label}: S3ListObjects=${lists}, expected 0 (row: ${row})" +} + +check_count() +{ + local label=$1 + local name=$2 + local got=$3 + local mode=$4 + case "$mode" in + eq0) [[ $got == "0" ]] || fail "${label}: ${name}=${got}, expected 0" ;; + ge1) [[ $got -ge 1 ]] || fail "${label}: ${name}=${got}, expected >= 1" ;; + any) ;; + *) fail "${label}: bad mode ${mode}" ;; + esac +} + +${CLICKHOUSE_CLIENT} -q " + INSERT INTO FUNCTION s3('${URL_LOCALHOST}', 'test', 'testtest', 'TSV', 'n UInt64') + SELECT 42 + SETTINGS s3_truncate_on_insert = 1" || fail "insert failed" + +# Drop whatever the INSERT cached. Each endpoint must miss on its own first read. +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear failed" +read_and_check localhost_cold "$URL_LOCALHOST" ge1 eq0 ge1 +read_and_check loopback_cold "$URL_LOOPBACK" ge1 eq0 ge1 +read_and_check loopback_warm "$URL_LOOPBACK" eq0 ge1 eq0 +read_and_check localhost_warm "$URL_LOCALHOST" eq0 ge1 eq0 + +echo OK diff --git a/tests/queries/0_stateless/05486_object_storage_identity_cache_overwrite.reference b/tests/queries/0_stateless/05486_object_storage_identity_cache_overwrite.reference new file mode 100644 index 0000000000000..d86bac9de59ab --- /dev/null +++ b/tests/queries/0_stateless/05486_object_storage_identity_cache_overwrite.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/05486_object_storage_identity_cache_overwrite.sh b/tests/queries/0_stateless/05486_object_storage_identity_cache_overwrite.sh new file mode 100755 index 0000000000000..119bcaff8f32f --- /dev/null +++ b/tests/queries/0_stateless/05486_object_storage_identity_cache_overwrite.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-parallel-replicas +# Tag no-fasttest: reads a file through the mock S3 server at localhost:11111 +# Tag no-parallel: the identity cache is process-global, and another test may clear it +# Tag no-parallel-replicas: per-query S3 request counts change when a read is spread across replicas + +# Overwriting an object drops its identity-cache entry. The next read misses and HEADs, so it +# sees the new size. The second insert is longer than the first. s3_validate_etag_on_read is off, +# so a stale cached size would be the read bound and the new rows would be truncated. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +URL="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" +READ_SETTINGS="max_threads = 1, use_hive_partitioning = 0, parallel_replicas_for_cluster_engines = 0, enable_filesystem_cache = 0, use_page_cache_for_object_storage = 0, s3_validate_etag_on_read = 0, use_object_storage_list_objects_cache = 0" + +fail() +{ + echo "FAIL: $*" >&2 + exit 1 +} + +# $1 label, $2 expected rows (tab-separated query result), $3 head mode, $4 hits mode, $5 misses mode +read_and_check() +{ + local label=$1 + local expected=$2 + local head_mode=$3 + local hits_mode=$4 + local misses_mode=$5 + local qid="idcache_ow_${label}_${CLICKHOUSE_DATABASE}" + local data row head hits misses lists + + data=$(${CLICKHOUSE_CLIENT} --query_id "$qid" -q \ + "SELECT n FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') ORDER BY n SETTINGS ${READ_SETTINGS} FORMAT TabSeparated") \ + || fail "${label}: select failed" + [[ $data == "$expected" ]] || fail "${label}: data '${data}', expected '${expected}'" + + ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "${label}: flush logs failed" + row=$(${CLICKHOUSE_CLIENT} -q " + SELECT + ProfileEvents['S3HeadObject'], + ProfileEvents['ObjectStorageIdentityCacheHits'], + ProfileEvents['ObjectStorageIdentityCacheMisses'], + ProfileEvents['S3ListObjects'] + FROM system.query_log + WHERE event_date >= yesterday() + AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND query_id = '${qid}' + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC + LIMIT 1 + FORMAT TabSeparated") || fail "${label}: query_log failed" + [[ -n $row ]] || fail "${label}: no query_log row" + + IFS=$'\t' read -r head hits misses lists <<< "$row" + check_count "$label" S3HeadObject "$head" "$head_mode" + check_count "$label" Hits "$hits" "$hits_mode" + check_count "$label" Misses "$misses" "$misses_mode" + [[ $lists == "0" ]] || fail "${label}: S3ListObjects=${lists}, expected 0 (row: ${row})" +} + +check_count() +{ + local label=$1 + local name=$2 + local got=$3 + local mode=$4 + case "$mode" in + eq0) [[ $got == "0" ]] || fail "${label}: ${name}=${got}, expected 0" ;; + ge1) [[ $got -ge 1 ]] || fail "${label}: ${name}=${got}, expected >= 1" ;; + *) fail "${label}: bad mode ${mode}" ;; + esac +} + +insert_values() +{ + local values=$1 + ${CLICKHOUSE_CLIENT} -q " + INSERT INTO FUNCTION s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') + ${values} + SETTINGS s3_truncate_on_insert = 1" || fail "insert failed: ${values}" +} + +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear failed" + +insert_values "SELECT 1" +read_and_check short_cold $'1' ge1 eq0 ge1 +read_and_check short_warm $'1' eq0 ge1 eq0 + +# No CLEAR. The upload drops the cached identity, so the next read misses and returns every new row. +insert_values "SELECT 10 UNION ALL SELECT 20 UNION ALL SELECT 30" +read_and_check overwritten $'10\n20\n30' ge1 eq0 ge1 +read_and_check overwritten_warm $'10\n20\n30' eq0 ge1 eq0 + +echo OK diff --git a/tests/queries/0_stateless/05488_object_storage_identity_cache_stale_store.reference b/tests/queries/0_stateless/05488_object_storage_identity_cache_stale_store.reference new file mode 100644 index 0000000000000..d86bac9de59ab --- /dev/null +++ b/tests/queries/0_stateless/05488_object_storage_identity_cache_stale_store.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/05488_object_storage_identity_cache_stale_store.sh b/tests/queries/0_stateless/05488_object_storage_identity_cache_stale_store.sh new file mode 100755 index 0000000000000..99b5c6c11a035 --- /dev/null +++ b/tests/queries/0_stateless/05488_object_storage_identity_cache_stale_store.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-parallel-replicas +# Tag no-fasttest: reads a file through the mock S3 server at localhost:11111 +# Tag no-parallel: the identity cache and the pause failpoint are process-global +# Tag no-parallel-replicas: per-query S3 request counts change when a read is spread across replicas + +# A metadata read is paused after its HEAD and before it stores the identity. An overwrite in that +# window advances the cache epoch. When the read resumes it must not store the identity it fetched, +# so the next read misses and returns the new rows. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +URL="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" +READ_SETTINGS="max_threads = 1, use_hive_partitioning = 0, parallel_replicas_for_cluster_engines = 0, enable_filesystem_cache = 0, use_page_cache_for_object_storage = 0, s3_validate_etag_on_read = 0, use_object_storage_list_objects_cache = 0" +FAILPOINT="object_storage_identity_cache_pause_before_store" + +fail() +{ + echo "FAIL: $*" >&2 + exit 1 +} + +cleanup() +{ + ${CLICKHOUSE_CLIENT} -q "SYSTEM DISABLE FAILPOINT ${FAILPOINT}" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +check_count() +{ + local label=$1 + local name=$2 + local got=$3 + local mode=$4 + case "$mode" in + eq0) [[ $got == "0" ]] || fail "${label}: ${name}=${got}, expected 0" ;; + ge1) [[ $got -ge 1 ]] || fail "${label}: ${name}=${got}, expected >= 1" ;; + *) fail "${label}: bad mode ${mode}" ;; + esac +} + +insert_values() +{ + local values=$1 + ${CLICKHOUSE_CLIENT} -q " + INSERT INTO FUNCTION s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') + ${values} + SETTINGS s3_truncate_on_insert = 1" || fail "insert failed: ${values}" +} + +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear failed" + +insert_values "SELECT 1" + +${CLICKHOUSE_CLIENT} -q "SYSTEM ENABLE FAILPOINT ${FAILPOINT}" || fail "enable failpoint failed" + +# The read misses, HEADs the current object, and pauses before storing that identity. +${CLICKHOUSE_CLIENT} -q \ + "SELECT n FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') ORDER BY n SETTINGS ${READ_SETTINGS} FORMAT TabSeparated" \ + >/dev/null & +reader_pid=$! + +${CLICKHOUSE_CLIENT} -q "SYSTEM WAIT FAILPOINT ${FAILPOINT} PAUSE" || fail "wait for pause failed" + +insert_values "SELECT 10 UNION ALL SELECT 20 UNION ALL SELECT 30" + +${CLICKHOUSE_CLIENT} -q "SYSTEM DISABLE FAILPOINT ${FAILPOINT}" || fail "disable failpoint failed" +wait "$reader_pid" || fail "in-flight read failed" + +qid="idcache_stale_${CLICKHOUSE_DATABASE}" +data=$(${CLICKHOUSE_CLIENT} --query_id "$qid" -q \ + "SELECT n FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') ORDER BY n SETTINGS ${READ_SETTINGS} FORMAT TabSeparated") \ + || fail "check select failed" +[[ $data == $'10\n20\n30' ]] || fail "data '${data}', expected the three new rows" + +${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "flush logs failed" +row=$(${CLICKHOUSE_CLIENT} -q " + SELECT + ProfileEvents['S3HeadObject'], + ProfileEvents['ObjectStorageIdentityCacheHits'], + ProfileEvents['ObjectStorageIdentityCacheMisses'] + FROM system.query_log + WHERE event_date >= yesterday() + AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND query_id = '${qid}' + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC + LIMIT 1 + FORMAT TabSeparated") || fail "query_log failed" +[[ -n $row ]] || fail "no query_log row" + +IFS=$'\t' read -r head hits misses <<< "$row" +check_count check S3HeadObject "$head" ge1 +check_count check Hits "$hits" eq0 +check_count check Misses "$misses" ge1 + +echo OK diff --git a/tests/queries/0_stateless/05489_object_storage_identity_cache_clear_during_fetch.reference b/tests/queries/0_stateless/05489_object_storage_identity_cache_clear_during_fetch.reference new file mode 100644 index 0000000000000..d86bac9de59ab --- /dev/null +++ b/tests/queries/0_stateless/05489_object_storage_identity_cache_clear_during_fetch.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/05489_object_storage_identity_cache_clear_during_fetch.sh b/tests/queries/0_stateless/05489_object_storage_identity_cache_clear_during_fetch.sh new file mode 100755 index 0000000000000..0cc7405e05bbe --- /dev/null +++ b/tests/queries/0_stateless/05489_object_storage_identity_cache_clear_during_fetch.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-parallel-replicas +# Tag no-fasttest: reads a file through the mock S3 server at localhost:11111 +# Tag no-parallel: the identity cache and the pause failpoint are process-global +# Tag no-parallel-replicas: per-query S3 request counts change when a read is spread across replicas + +# A metadata read is paused after its HEAD and before it stores the identity. The object is overwritten +# and then the whole cache is dropped. The drop advances the cache epoch, so the read must not store +# looks untouched again. When the read resumes it must still not store the identity it fetched before +# the overwrite: the next read misses and returns the new rows. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +URL="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" +READ_SETTINGS="max_threads = 1, use_hive_partitioning = 0, parallel_replicas_for_cluster_engines = 0, enable_filesystem_cache = 0, use_page_cache_for_object_storage = 0, s3_validate_etag_on_read = 0, use_object_storage_list_objects_cache = 0" +FAILPOINT="object_storage_identity_cache_pause_before_store" + +fail() +{ + echo "FAIL: $*" >&2 + exit 1 +} + +cleanup() +{ + ${CLICKHOUSE_CLIENT} -q "SYSTEM DISABLE FAILPOINT ${FAILPOINT}" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +check_count() +{ + local label=$1 + local name=$2 + local got=$3 + local mode=$4 + case "$mode" in + eq0) [[ $got == "0" ]] || fail "${label}: ${name}=${got}, expected 0" ;; + ge1) [[ $got -ge 1 ]] || fail "${label}: ${name}=${got}, expected >= 1" ;; + *) fail "${label}: bad mode ${mode}" ;; + esac +} + +insert_values() +{ + local values=$1 + ${CLICKHOUSE_CLIENT} -q " + INSERT INTO FUNCTION s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') + ${values} + SETTINGS s3_truncate_on_insert = 1" || fail "insert failed: ${values}" +} + +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear failed" + +insert_values "SELECT 1" + +${CLICKHOUSE_CLIENT} -q "SYSTEM ENABLE FAILPOINT ${FAILPOINT}" || fail "enable failpoint failed" + +# The read misses, HEADs the current object, and pauses before storing that identity. +${CLICKHOUSE_CLIENT} -q \ + "SELECT n FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') ORDER BY n SETTINGS ${READ_SETTINGS} FORMAT TabSeparated" \ + >/dev/null & +reader_pid=$! + +${CLICKHOUSE_CLIENT} -q "SYSTEM WAIT FAILPOINT ${FAILPOINT} PAUSE" || fail "wait for pause failed" + +insert_values "SELECT 10 UNION ALL SELECT 20 UNION ALL SELECT 30" + +# Drops every entry and advances the epoch while the read is still paused. +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear during fetch failed" + +${CLICKHOUSE_CLIENT} -q "SYSTEM DISABLE FAILPOINT ${FAILPOINT}" || fail "disable failpoint failed" +wait "$reader_pid" || fail "in-flight read failed" + +qid="idcache_clear_${CLICKHOUSE_DATABASE}" +data=$(${CLICKHOUSE_CLIENT} --query_id "$qid" -q \ + "SELECT n FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') ORDER BY n SETTINGS ${READ_SETTINGS} FORMAT TabSeparated") \ + || fail "check select failed" +[[ $data == $'10\n20\n30' ]] || fail "data '${data}', expected the three new rows" + +${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "flush logs failed" +row=$(${CLICKHOUSE_CLIENT} -q " + SELECT + ProfileEvents['S3HeadObject'], + ProfileEvents['ObjectStorageIdentityCacheHits'], + ProfileEvents['ObjectStorageIdentityCacheMisses'] + FROM system.query_log + WHERE event_date >= yesterday() + AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND query_id = '${qid}' + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC + LIMIT 1 + FORMAT TabSeparated") || fail "query_log failed" +[[ -n $row ]] || fail "no query_log row" + +IFS=$'\t' read -r head hits misses <<< "$row" +check_count check S3HeadObject "$head" ge1 +check_count check Hits "$hits" eq0 +check_count check Misses "$misses" ge1 + +echo OK diff --git a/tests/queries/0_stateless/05490_object_storage_identity_cache_external_replace.reference b/tests/queries/0_stateless/05490_object_storage_identity_cache_external_replace.reference new file mode 100644 index 0000000000000..d86bac9de59ab --- /dev/null +++ b/tests/queries/0_stateless/05490_object_storage_identity_cache_external_replace.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/05490_object_storage_identity_cache_external_replace.sh b/tests/queries/0_stateless/05490_object_storage_identity_cache_external_replace.sh new file mode 100755 index 0000000000000..4c4ab17875eff --- /dev/null +++ b/tests/queries/0_stateless/05490_object_storage_identity_cache_external_replace.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-parallel-replicas +# Tag no-fasttest: reads a file through the mock S3 server at localhost:11111 +# Tag no-parallel: the identity cache is process-global, and another test may clear it +# Tag no-parallel-replicas: per-query S3 request counts change when a read is spread across replicas + +# A replace that does not go through this server leaves the cached size and ETag in place. +# The next read is pinned to that ETag, fails with S3_OBJECT_CHANGED_DURING_READ, and must +# drop the entry. The retry misses, HEADs, and returns the new rows. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +URL="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" +READ_SETTINGS="max_threads = 1, use_hive_partitioning = 0, parallel_replicas_for_cluster_engines = 0, enable_filesystem_cache = 0, use_page_cache_for_object_storage = 0, s3_validate_etag_on_read = 1, use_object_storage_list_objects_cache = 0" + +fail() +{ + echo "FAIL: $*" >&2 + exit 1 +} + +# $1 label, $2 expected rows (tab-separated query result), $3 head mode, $4 hits mode, $5 misses mode +read_and_check() +{ + local label=$1 + local expected=$2 + local head_mode=$3 + local hits_mode=$4 + local misses_mode=$5 + local qid="idcache_ext_${label}_${CLICKHOUSE_DATABASE}" + local data row head hits misses lists + + data=$(${CLICKHOUSE_CLIENT} --query_id "$qid" -q \ + "SELECT n FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') ORDER BY n SETTINGS ${READ_SETTINGS} FORMAT TabSeparated") \ + || fail "${label}: select failed" + [[ $data == "$expected" ]] || fail "${label}: data '${data}', expected '${expected}'" + + ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "${label}: flush logs failed" + row=$(${CLICKHOUSE_CLIENT} -q " + SELECT + ProfileEvents['S3HeadObject'], + ProfileEvents['ObjectStorageIdentityCacheHits'], + ProfileEvents['ObjectStorageIdentityCacheMisses'], + ProfileEvents['S3ListObjects'] + FROM system.query_log + WHERE event_date >= yesterday() + AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND query_id = '${qid}' + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC + LIMIT 1 + FORMAT TabSeparated") || fail "${label}: query_log failed" + [[ -n $row ]] || fail "${label}: no query_log row" + + IFS=$'\t' read -r head hits misses lists <<< "$row" + check_count "$label" S3HeadObject "$head" "$head_mode" + check_count "$label" Hits "$hits" "$hits_mode" + check_count "$label" Misses "$misses" "$misses_mode" + [[ $lists == "0" ]] || fail "${label}: S3ListObjects=${lists}, expected 0 (row: ${row})" +} + +check_count() +{ + local label=$1 + local name=$2 + local got=$3 + local mode=$4 + case "$mode" in + eq0) [[ $got == "0" ]] || fail "${label}: ${name}=${got}, expected 0" ;; + ge1) [[ $got -ge 1 ]] || fail "${label}: ${name}=${got}, expected >= 1" ;; + *) fail "${label}: bad mode ${mode}" ;; + esac +} + +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear failed" + +${CLICKHOUSE_CLIENT} -q " + INSERT INTO FUNCTION s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') + SELECT 1 + SETTINGS s3_truncate_on_insert = 1" || fail "insert failed" + +read_and_check cold $'1' ge1 eq0 ge1 +read_and_check warm $'1' eq0 ge1 eq0 + +# This PUT does not go through S3ObjectStorage::writeObject, so nothing drops the cached identity. +payload="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" +printf '10\n20\n30\n' > "$payload" +curl -fsS -X PUT "$URL" --upload-file "$payload" >/dev/null || fail "external put failed" + +stale_err=$(${CLICKHOUSE_CLIENT} -q \ + "SELECT n FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') ORDER BY n SETTINGS ${READ_SETTINGS} FORMAT TabSeparated" 2>&1) \ + && fail "stale: the replaced object was read" +[[ $stale_err == *"S3_OBJECT_CHANGED_DURING_READ"* ]] || fail "stale: unexpected error: ${stale_err}" + +read_and_check retried $'10\n20\n30' ge1 eq0 ge1 +read_and_check retried_warm $'10\n20\n30' eq0 ge1 eq0 + +echo OK diff --git a/tests/queries/0_stateless/05491_url_object_metadata_shard.reference b/tests/queries/0_stateless/05491_url_object_metadata_shard.reference new file mode 100644 index 0000000000000..d86bac9de59ab --- /dev/null +++ b/tests/queries/0_stateless/05491_url_object_metadata_shard.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/05491_url_object_metadata_shard.sh b/tests/queries/0_stateless/05491_url_object_metadata_shard.sh new file mode 100755 index 0000000000000..308f9fdee0292 --- /dev/null +++ b/tests/queries/0_stateless/05491_url_object_metadata_shard.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: serves two local HTTP shards so url() can list the same path on each. + +# Two url() shards expose the same relative path with different sizes. Metadata for each listed +# file must be read from that file's shard. Losing the shard index HEADs the first shard for both +# and uses its size as the read bound, so the larger file is truncated. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +fail() +{ + echo "FAIL: $*" >&2 + exit 1 +} + +free_port() +{ + python3 -c " +import socket +s = socket.socket() +s.bind(('127.0.0.1', 0)) +print(s.getsockname()[1]) +s.close() +" +} + +PORT_A=$(free_port) +PORT_B=$(free_port) +[[ -n $PORT_A && -n $PORT_B && $PORT_A != "$PORT_B" ]] || fail "could not reserve two ports" + +python3 -c " +from http.server import HTTPServer, BaseHTTPRequestHandler +import threading + +SMALL = b'1\n' +LARGE = b'10\n20\n30\n' +INDEX = b'file.tsv\n' + +def make_handler(body): + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def respond(self, with_body): + path = self.path.split('?')[0] + if path == '/data/': + data, content_type = INDEX, 'text/html' + elif path == '/data/file.tsv': + data, content_type = body, 'text/plain' + else: + self.send_response(404) + self.end_headers() + return + # A known size makes url() send Range: bytes=0-(size-1). A 200 for that request is logged + # as a warning, and the stateless runner fails the test on any client stderr. + total = len(data) + start, end, ranged = 0, total - 1, False + range_header = self.headers.get('Range') if path == '/data/file.tsv' else None + if range_header and total: + spec = range_header.split('=', 1)[-1].split(',')[0].strip() + left, _, right = spec.partition('-') + if left: + start = int(left) + if right: + end = int(right) + if start < 0 or end < start or start >= total: + self.send_response(416) + self.send_header('Content-Range', 'bytes */' + str(total)) + self.end_headers() + return + end = min(end, total - 1) + data = data[start:end + 1] + ranged = True + self.send_response(206 if ranged else 200) + self.send_header('Content-Type', content_type) + self.send_header('Accept-Ranges', 'bytes') + self.send_header('Content-Length', str(len(data))) + if ranged: + self.send_header('Content-Range', 'bytes ' + str(start) + '-' + str(end) + '/' + str(total)) + self.end_headers() + if with_body: + self.wfile.write(data) + + def do_HEAD(self): + self.respond(False) + + def do_GET(self): + self.respond(True) + return Handler + +def serve(port, body): + HTTPServer(('127.0.0.1', port), make_handler(body)).serve_forever() + +threading.Thread(target=serve, args=(${PORT_A}, SMALL), daemon=True).start() +serve(${PORT_B}, LARGE) +" & +HTTP_PID=$! +trap 'kill $HTTP_PID 2>/dev/null; wait $HTTP_PID 2>/dev/null' EXIT + +for _ in $(seq 1 50); do + curl -sf "http://127.0.0.1:${PORT_A}/data/" >/dev/null && curl -sf "http://127.0.0.1:${PORT_B}/data/" >/dev/null && break + sleep 0.1 +done +curl -sf "http://127.0.0.1:${PORT_A}/data/" >/dev/null || fail "shard A did not start" +curl -sf "http://127.0.0.1:${PORT_B}/data/" >/dev/null || fail "shard B did not start" + +URL="http://127.0.0.1:{${PORT_A},${PORT_B}}/data/*.tsv" +data=$(${CLICKHOUSE_CLIENT} -q " + SET allow_experimental_url_wildcard_from_index_pages = 1; + SELECT n + FROM url('${URL}', 'TSV', 'n UInt64') + ORDER BY n + SETTINGS max_threads = 1, http_max_tries = 1 + FORMAT TabSeparated") || fail "select failed" + +expected=$'1\n10\n20\n30' +[[ $data == "$expected" ]] || fail "data '${data}', expected '${expected}'" + +echo OK diff --git a/tests/queries/0_stateless/05492_object_storage_identity_cache_last_modified.reference b/tests/queries/0_stateless/05492_object_storage_identity_cache_last_modified.reference new file mode 100644 index 0000000000000..d86bac9de59ab --- /dev/null +++ b/tests/queries/0_stateless/05492_object_storage_identity_cache_last_modified.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/05492_object_storage_identity_cache_last_modified.sh b/tests/queries/0_stateless/05492_object_storage_identity_cache_last_modified.sh new file mode 100755 index 0000000000000..2ded1d045ca58 --- /dev/null +++ b/tests/queries/0_stateless/05492_object_storage_identity_cache_last_modified.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-parallel-replicas +# Tag no-fasttest: reads a file through the mock S3 server at localhost:11111 +# Tag no-parallel: the identity cache is process-global, and another test may clear it +# Tag no-parallel-replicas: per-query S3 request counts change when a read is spread across replicas + +# A cache hit must return the Last-Modified stored with the identity. Otherwise the second read of +# the same object reports `_time` as NULL. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +URL="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" +READ_SETTINGS="max_threads = 1, use_hive_partitioning = 0, parallel_replicas_for_cluster_engines = 0, enable_filesystem_cache = 0, use_page_cache_for_object_storage = 0, s3_validate_etag_on_read = 0, use_object_storage_list_objects_cache = 0" + +fail() +{ + echo "FAIL: $*" >&2 + exit 1 +} + +# $1 label, $2 hits mode, $3 misses mode. Prints the _time value. +read_time() +{ + local label=$1 + local hits_mode=$2 + local misses_mode=$3 + local qid="idcache_time_${label}_${CLICKHOUSE_DATABASE}" + local value row head hits misses + + value=$(${CLICKHOUSE_CLIENT} --query_id "$qid" -q " + SELECT if(isNull(_time), 'NULL', toString(_time)) + FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') + SETTINGS ${READ_SETTINGS} + FORMAT TabSeparated") || fail "${label}: select failed" + + ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "${label}: flush logs failed" + row=$(${CLICKHOUSE_CLIENT} -q " + SELECT + ProfileEvents['S3HeadObject'], + ProfileEvents['ObjectStorageIdentityCacheHits'], + ProfileEvents['ObjectStorageIdentityCacheMisses'] + FROM system.query_log + WHERE event_date >= yesterday() + AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND query_id = '${qid}' + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC + LIMIT 1 + FORMAT TabSeparated") || fail "${label}: query_log failed" + [[ -n $row ]] || fail "${label}: no query_log row" + + IFS=$'\t' read -r head hits misses <<< "$row" + case "$hits_mode" in + eq0) [[ $hits == "0" ]] || fail "${label}: Hits=${hits}, expected 0" ;; + ge1) + [[ $hits -ge 1 ]] || fail "${label}: Hits=${hits}, expected >= 1" + [[ $head == "0" ]] || fail "${label}: cache hit still sent S3HeadObject=${head}" + ;; + esac + case "$misses_mode" in + eq0) [[ $misses == "0" ]] || fail "${label}: Misses=${misses}, expected 0" ;; + ge1) + [[ $misses -ge 1 ]] || fail "${label}: Misses=${misses}, expected >= 1" + [[ $head -ge 1 ]] || fail "${label}: S3HeadObject=${head}, expected >= 1" + ;; + esac + + printf '%s' "$value" +} + +${CLICKHOUSE_CLIENT} -q " + INSERT INTO FUNCTION s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') + SELECT 1 + SETTINGS s3_truncate_on_insert = 1" || fail "insert failed" + +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear failed" + +cold=$(read_time cold eq0 ge1) +warm=$(read_time warm ge1 eq0) + +[[ $cold != "NULL" && -n $cold ]] || fail "cold _time is '${cold}'" +[[ $warm == "$cold" ]] || fail "warm _time '${warm}', cold '${cold}'" + +echo OK diff --git a/tests/queries/0_stateless/05493_object_storage_identity_cache_count_after_replace.reference b/tests/queries/0_stateless/05493_object_storage_identity_cache_count_after_replace.reference new file mode 100644 index 0000000000000..d86bac9de59ab --- /dev/null +++ b/tests/queries/0_stateless/05493_object_storage_identity_cache_count_after_replace.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/05493_object_storage_identity_cache_count_after_replace.sh b/tests/queries/0_stateless/05493_object_storage_identity_cache_count_after_replace.sh new file mode 100755 index 0000000000000..1353102740abe --- /dev/null +++ b/tests/queries/0_stateless/05493_object_storage_identity_cache_count_after_replace.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-parallel-replicas +# Tag no-fasttest: reads a file through the mock S3 server at localhost:11111 +# Tag no-parallel: the identity cache and the schema cache are process-global +# Tag no-parallel-replicas: per-query S3 request counts change when a read is spread across replicas + +# An identity-cache hit returns the stored Last-Modified for `_time`, but that time must not prove +# the object is unchanged. Otherwise `count()` keeps the previous row count after an external replace. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +URL="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" +READ_SETTINGS="max_threads = 1, use_hive_partitioning = 0, parallel_replicas_for_cluster_engines = 0, enable_filesystem_cache = 0, use_page_cache_for_object_storage = 0, s3_validate_etag_on_read = 1, use_object_storage_list_objects_cache = 0, use_cache_for_count_from_files = 1, schema_inference_use_cache_for_s3 = 1" + +fail() +{ + echo "FAIL: $*" >&2 + exit 1 +} + +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear failed" + +${CLICKHOUSE_CLIENT} -q " + INSERT INTO FUNCTION s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') + SELECT 1 + SETTINGS s3_truncate_on_insert = 1" || fail "insert failed" + +# Last-Modified has second resolution. The count is registered in the same second as the upload unless +# we wait, and a time equal to the registration does not reproduce the stale-count hit. +sleep 1 + +count1=$(${CLICKHOUSE_CLIENT} -q " + SELECT count() FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') + SETTINGS ${READ_SETTINGS}") || fail "first count failed" +[[ $count1 == "1" ]] || fail "first count '${count1}', expected 1" + +payload="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" +printf '10\n20\n30\n' > "$payload" +curl -fsS -X PUT "$URL" --upload-file "$payload" >/dev/null || fail "external put failed" + +stale_err=$(${CLICKHOUSE_CLIENT} -q " + SELECT count() FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') + SETTINGS ${READ_SETTINGS}" 2>&1) && fail "replaced object: count succeeded with '${stale_err}'" +[[ $stale_err == *"S3_OBJECT_CHANGED_DURING_READ"* ]] || fail "replaced object: unexpected error: ${stale_err}" + +count2=$(${CLICKHOUSE_CLIENT} -q " + SELECT count() FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') + SETTINGS ${READ_SETTINGS}") || fail "retry count failed" +[[ $count2 == "3" ]] || fail "retry count '${count2}', expected 3" + +echo OK diff --git a/tests/queries/0_stateless/05494_object_storage_identity_cache_filesystem_cache_replace.reference b/tests/queries/0_stateless/05494_object_storage_identity_cache_filesystem_cache_replace.reference new file mode 100644 index 0000000000000..d86bac9de59ab --- /dev/null +++ b/tests/queries/0_stateless/05494_object_storage_identity_cache_filesystem_cache_replace.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/05494_object_storage_identity_cache_filesystem_cache_replace.sh b/tests/queries/0_stateless/05494_object_storage_identity_cache_filesystem_cache_replace.sh new file mode 100755 index 0000000000000..7e4883a5dd7b8 --- /dev/null +++ b/tests/queries/0_stateless/05494_object_storage_identity_cache_filesystem_cache_replace.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-parallel-replicas +# Tag no-fasttest: reads a file through the mock S3 server at localhost:11111 +# Tag no-parallel: the identity cache and the filesystem cache are process-global +# Tag no-parallel-replicas: per-query S3 request counts change when a read is spread across replicas + +# A filesystem-cache hit does not issue a GET, so it cannot see that an object was replaced. The read +# must revalidate the identity before using its ETag as the cache key. An unchanged object is still +# served from the cache. A replaced object is read again. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +URL="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" +# Pin populate-on-miss. The stateless runner can set +# `read_from_filesystem_cache_if_exists_otherwise_bypass_cache` to 1, and then a miss is read from +# S3 and not stored, so the second read still issues a GET. +READ_SETTINGS="max_threads = 1, use_hive_partitioning = 0, parallel_replicas_for_cluster_engines = 0, enable_filesystem_cache = 1, filesystem_cache_name = 'cache_for_readbigat', read_from_filesystem_cache_if_exists_otherwise_bypass_cache = 0, use_page_cache_for_object_storage = 0, s3_validate_etag_on_read = 1, use_object_storage_list_objects_cache = 0" + +fail() +{ + echo "FAIL: $*" >&2 + exit 1 +} + +# $1 label, $2 expected rows, $3 head mode, $4 get mode, $5 cache-read mode +read_and_check() +{ + local label=$1 + local expected=$2 + local head_mode=$3 + local get_mode=$4 + local cache_mode=$5 + local qid="idcache_fscache_${label}_${CLICKHOUSE_DATABASE}" + local data row head gets cache_bytes + + data=$(${CLICKHOUSE_CLIENT} --query_id "$qid" -q \ + "SELECT n FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') ORDER BY n SETTINGS ${READ_SETTINGS} FORMAT TabSeparated") \ + || fail "${label}: select failed" + [[ $data == "$expected" ]] || fail "${label}: data '${data}', expected '${expected}'" + + ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "${label}: flush logs failed" + row=$(${CLICKHOUSE_CLIENT} -q " + SELECT + ProfileEvents['S3HeadObject'], + ProfileEvents['S3GetObject'], + ProfileEvents['CachedReadBufferReadFromCacheBytes'] + FROM system.query_log + WHERE event_date >= yesterday() + AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND query_id = '${qid}' + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC + LIMIT 1 + FORMAT TabSeparated") || fail "${label}: query_log failed" + [[ -n $row ]] || fail "${label}: no query_log row" + + IFS=$'\t' read -r head gets cache_bytes <<< "$row" + case "$head_mode" in + ge1) [[ $head -ge 1 ]] || fail "${label}: S3HeadObject=${head}, expected >= 1" ;; + *) fail "${label}: bad head mode ${head_mode}" ;; + esac + case "$get_mode" in + eq0) [[ $gets == "0" ]] || fail "${label}: S3GetObject=${gets}, expected 0" ;; + ge1) [[ $gets -ge 1 ]] || fail "${label}: S3GetObject=${gets}, expected >= 1" ;; + *) fail "${label}: bad get mode ${get_mode}" ;; + esac + case "$cache_mode" in + ge1) [[ $cache_bytes -ge 1 ]] || fail "${label}: CachedReadBufferReadFromCacheBytes=${cache_bytes}, expected >= 1" ;; + any) ;; + *) fail "${label}: bad cache mode ${cache_mode}" ;; + esac +} + +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear identity cache failed" +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR FILESYSTEM CACHE 'cache_for_readbigat'" || fail "clear filesystem cache failed" + +${CLICKHOUSE_CLIENT} -q " + INSERT INTO FUNCTION s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') + SELECT 1 + SETTINGS s3_truncate_on_insert = 1" || fail "insert failed" + +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear identity cache after insert failed" + +read_and_check cold $'1' ge1 ge1 any +read_and_check cached $'1' ge1 eq0 ge1 + +payload="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" +printf '10\n20\n30\n' > "$payload" +curl -fsS -X PUT "$URL" --upload-file "$payload" >/dev/null || fail "external put failed" + +read_and_check replaced $'10\n20\n30' ge1 ge1 any + +echo OK diff --git a/tests/queries/0_stateless/05497_object_storage_identity_cache_parquet_count_after_replace.reference b/tests/queries/0_stateless/05497_object_storage_identity_cache_parquet_count_after_replace.reference new file mode 100644 index 0000000000000..d86bac9de59ab --- /dev/null +++ b/tests/queries/0_stateless/05497_object_storage_identity_cache_parquet_count_after_replace.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/05497_object_storage_identity_cache_parquet_count_after_replace.sh b/tests/queries/0_stateless/05497_object_storage_identity_cache_parquet_count_after_replace.sh new file mode 100755 index 0000000000000..3c9425ca2ab64 --- /dev/null +++ b/tests/queries/0_stateless/05497_object_storage_identity_cache_parquet_count_after_replace.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-parallel-replicas +# Tag no-fasttest: reads a file through the mock S3 server at localhost:11111 +# Tag no-parallel: the identity cache and the parquet metadata cache are process-global +# Tag no-parallel-replicas: per-query S3 request counts change when a read is spread across replicas + +# A Parquet count reads the footer from the parquet metadata cache and does not GET. The cache key +# includes the object ETag. An identity-cache hit must not supply that ETag until a HEAD confirms it, +# or the count keeps the replaced object's row count. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +URL="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}.parquet" +READ_SETTINGS="max_threads = 1, use_hive_partitioning = 0, parallel_replicas_for_cluster_engines = 0, enable_filesystem_cache = 0, use_page_cache_for_object_storage = 0, s3_validate_etag_on_read = 1, use_object_storage_list_objects_cache = 0, use_cache_for_count_from_files = 1, schema_inference_use_cache_for_s3 = 1, use_parquet_metadata_cache = 1" + +fail() +{ + echo "FAIL: $*" >&2 + exit 1 +} + +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear identity cache failed" +${CLICKHOUSE_CLIENT} -q "SYSTEM DROP PARQUET METADATA CACHE" || fail "drop parquet metadata cache failed" + +${CLICKHOUSE_CLIENT} -q " + INSERT INTO FUNCTION s3('${URL}', 'test', 'testtest', 'Parquet', 'n UInt64') + SELECT toUInt64(1) + SETTINGS s3_truncate_on_insert = 1" || fail "insert failed" + +count1=$(${CLICKHOUSE_CLIENT} -q " + SELECT count() FROM s3('${URL}', 'test', 'testtest', 'Parquet', 'n UInt64') + SETTINGS ${READ_SETTINGS}") || fail "first count failed" +[[ $count1 == "1" ]] || fail "first count '${count1}', expected 1" + +payload="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}.parquet" +${CLICKHOUSE_CLIENT} -q "SELECT toUInt64(number) AS n FROM numbers(3) FORMAT Parquet" > "$payload" || fail "local parquet failed" +curl -fsS -X PUT "$URL" --upload-file "$payload" >/dev/null || fail "external put failed" + +replaced_qid="idcache_parquet_count_${CLICKHOUSE_DATABASE}" +count2=$(${CLICKHOUSE_CLIENT} --query_id "$replaced_qid" -q " + SELECT count() FROM s3('${URL}', 'test', 'testtest', 'Parquet', 'n UInt64') + SETTINGS ${READ_SETTINGS}") || fail "replaced count failed" +[[ $count2 == "3" ]] || fail "replaced count '${count2}', expected 3" + +${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "flush logs failed" +heads=$(${CLICKHOUSE_CLIENT} -q " + SELECT ProfileEvents['S3HeadObject'] + FROM system.query_log + WHERE event_date >= yesterday() + AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND query_id = '${replaced_qid}' + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC + LIMIT 1 + FORMAT TabSeparated") || fail "query_log failed" +[[ $heads -ge 1 ]] || fail "replaced count did not HEAD (S3HeadObject=${heads})" + +echo OK diff --git a/tests/queries/0_stateless/05498_object_storage_identity_cache_skip_replaced.reference b/tests/queries/0_stateless/05498_object_storage_identity_cache_skip_replaced.reference new file mode 100644 index 0000000000000..d86bac9de59ab --- /dev/null +++ b/tests/queries/0_stateless/05498_object_storage_identity_cache_skip_replaced.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/05498_object_storage_identity_cache_skip_replaced.sh b/tests/queries/0_stateless/05498_object_storage_identity_cache_skip_replaced.sh new file mode 100755 index 0000000000000..836216defedba --- /dev/null +++ b/tests/queries/0_stateless/05498_object_storage_identity_cache_skip_replaced.sh @@ -0,0 +1,197 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-parallel-replicas +# Tag no-fasttest: reads a file through the mock S3 server at localhost:11111 +# Tag no-parallel: the identity cache and the query condition cache are process-global +# Tag no-parallel-replicas: per-query S3 request counts change when a read is spread across replicas + +# An identity-cache hit still has the previous size and ETag. Two decisions use that metadata and +# then do not GET: s3_skip_empty_files (default on) skips a cached size of 0, and the query condition +# cache skips a file whose cached ETag matched no row group. An external replace must still be read. +# A condition-cache re-HEAD that finds an empty replacement must skip it, not open it. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +fail() +{ + echo "FAIL: $*" >&2 + exit 1 +} + +READ_SETTINGS="max_threads = 1, use_hive_partitioning = 0, parallel_replicas_for_cluster_engines = 0, enable_filesystem_cache = 0, use_page_cache_for_object_storage = 0, s3_validate_etag_on_read = 1, use_object_storage_list_objects_cache = 0, s3_skip_empty_files = 1" + +# --- A cached empty object must not hide a later non-empty replace. --- + +EMPTY_URL="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}_empty.tsv" + +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear identity cache failed" + +curl -fsS -X PUT "$EMPTY_URL" --data-binary '' >/dev/null || fail "empty put failed" + +empty_cold=$(${CLICKHOUSE_CLIENT} -q " + SELECT n FROM s3('${EMPTY_URL}', 'test', 'testtest', 'TSV', 'n UInt64') + SETTINGS ${READ_SETTINGS} FORMAT TabSeparated") || fail "empty cold select failed" +[[ -z $empty_cold ]] || fail "empty cold: data '${empty_cold}'" + +empty_warm=$(${CLICKHOUSE_CLIENT} -q " + SELECT n FROM s3('${EMPTY_URL}', 'test', 'testtest', 'TSV', 'n UInt64') + SETTINGS ${READ_SETTINGS} FORMAT TabSeparated") || fail "empty warm select failed" +[[ -z $empty_warm ]] || fail "empty warm: data '${empty_warm}'" + +empty_payload="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}_empty.tsv" +printf '7\n' > "$empty_payload" +curl -fsS -X PUT "$EMPTY_URL" --upload-file "$empty_payload" >/dev/null || fail "empty replace put failed" + +replaced_qid="idcache_skip_empty_${CLICKHOUSE_DATABASE}" +empty_replaced=$(${CLICKHOUSE_CLIENT} --query_id "$replaced_qid" -q " + SELECT n FROM s3('${EMPTY_URL}', 'test', 'testtest', 'TSV', 'n UInt64') + SETTINGS ${READ_SETTINGS} FORMAT TabSeparated") || fail "empty replaced select failed" +[[ $empty_replaced == "7" ]] || fail "empty replaced: data '${empty_replaced}', expected 7" + +${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "flush logs failed" +empty_heads=$(${CLICKHOUSE_CLIENT} -q " + SELECT ProfileEvents['S3HeadObject'] + FROM system.query_log + WHERE event_date >= yesterday() + AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND query_id = '${replaced_qid}' + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC + LIMIT 1 + FORMAT TabSeparated") || fail "empty query_log failed" +[[ $empty_heads -ge 1 ]] || fail "empty replaced did not HEAD (S3HeadObject=${empty_heads})" + +# --- A query-condition-cache hit on the previous ETag must not drop the replaced file. --- + +PARQUET_URL="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}_qcc.parquet" +TABLE="idcache_skip_qcc_${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear identity cache before parquet failed" +${CLICKHOUSE_CLIENT} -q "SYSTEM DROP QUERY CONDITION CACHE" || fail "drop query condition cache failed" +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ${TABLE}" || fail "drop table failed" + +${CLICKHOUSE_CLIENT} -q " + CREATE TABLE ${TABLE} (n UInt64) + ENGINE = S3('${PARQUET_URL}', 'test', 'testtest', 'Parquet')" || fail "create table failed" + +${CLICKHOUSE_CLIENT} -q " + INSERT INTO ${TABLE} SELECT toUInt64(1) + SETTINGS s3_truncate_on_insert = 1" || fail "parquet insert failed" + +FILTER_SETTINGS="${READ_SETTINGS}, use_query_condition_cache = 1" + +qcc_first=$(${CLICKHOUSE_CLIENT} -q " + SELECT count() FROM ${TABLE} WHERE n = 3 SETTINGS ${FILTER_SETTINGS}") || fail "qcc first select failed" +[[ $qcc_first == "0" ]] || fail "qcc first: '${qcc_first}', expected 0" + +# The second read must hit the condition cache, or the replace below does not exercise the skip. +qcc_second_qid="idcache_skip_qcc_warm_${CLICKHOUSE_DATABASE}" +qcc_second=$(${CLICKHOUSE_CLIENT} --query_id "$qcc_second_qid" -q " + SELECT count() FROM ${TABLE} WHERE n = 3 SETTINGS ${FILTER_SETTINGS}") || fail "qcc warm select failed" +[[ $qcc_second == "0" ]] || fail "qcc warm: '${qcc_second}', expected 0" + +${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "flush logs before qcc hit check failed" +qcc_hits=$(${CLICKHOUSE_CLIENT} -q " + SELECT ProfileEvents['QueryConditionCacheHits'] + FROM system.query_log + WHERE event_date >= yesterday() + AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND query_id = '${qcc_second_qid}' + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC + LIMIT 1 + FORMAT TabSeparated") || fail "qcc warm query_log failed" +[[ $qcc_hits -ge 1 ]] || fail "qcc warm was not a condition-cache hit (QueryConditionCacheHits=${qcc_hits})" + +parquet_payload="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}_qcc.parquet" +${CLICKHOUSE_CLIENT} -q "SELECT toUInt64(3) AS n FORMAT Parquet" > "$parquet_payload" || fail "local parquet failed" +curl -fsS -X PUT "$PARQUET_URL" --upload-file "$parquet_payload" >/dev/null || fail "parquet replace put failed" + +qcc_replaced_qid="idcache_skip_qcc_replaced_${CLICKHOUSE_DATABASE}" +qcc_replaced=$(${CLICKHOUSE_CLIENT} --query_id "$qcc_replaced_qid" -q " + SELECT count() FROM ${TABLE} WHERE n = 3 SETTINGS ${FILTER_SETTINGS}") || fail "qcc replaced select failed" +[[ $qcc_replaced == "1" ]] || fail "qcc replaced: '${qcc_replaced}', expected 1" + +${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "flush logs after qcc replace failed" +qcc_heads=$(${CLICKHOUSE_CLIENT} -q " + SELECT ProfileEvents['S3HeadObject'] + FROM system.query_log + WHERE event_date >= yesterday() + AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND query_id = '${qcc_replaced_qid}' + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC + LIMIT 1 + FORMAT TabSeparated") || fail "qcc replaced query_log failed" +[[ $qcc_heads -ge 1 ]] || fail "qcc replaced did not HEAD (S3HeadObject=${qcc_heads})" + +${CLICKHOUSE_CLIENT} -q "DROP TABLE ${TABLE}" || fail "drop table at end failed" + +# --- A condition-cache re-HEAD that finds an empty replacement must skip it, not open it. --- + +EMPTY_PARQUET_URL="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}_qcc_empty.parquet" +EMPTY_TABLE="idcache_skip_qcc_empty_${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear identity cache before empty parquet failed" +${CLICKHOUSE_CLIENT} -q "SYSTEM DROP QUERY CONDITION CACHE" || fail "drop query condition cache before empty parquet failed" +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ${EMPTY_TABLE}" || fail "drop empty parquet table failed" + +${CLICKHOUSE_CLIENT} -q " + CREATE TABLE ${EMPTY_TABLE} (n UInt64) + ENGINE = S3('${EMPTY_PARQUET_URL}', 'test', 'testtest', 'Parquet')" || fail "create empty parquet table failed" + +${CLICKHOUSE_CLIENT} -q " + INSERT INTO ${EMPTY_TABLE} SELECT toUInt64(1) + SETTINGS s3_truncate_on_insert = 1" || fail "empty parquet insert failed" + +empty_qcc_first=$(${CLICKHOUSE_CLIENT} -q " + SELECT count() FROM ${EMPTY_TABLE} WHERE n = 3 SETTINGS ${FILTER_SETTINGS}") || fail "empty parquet first select failed" +[[ $empty_qcc_first == "0" ]] || fail "empty parquet first: '${empty_qcc_first}', expected 0" + +empty_qcc_warm_qid="idcache_skip_qcc_empty_warm_${CLICKHOUSE_DATABASE}" +empty_qcc_warm=$(${CLICKHOUSE_CLIENT} --query_id "$empty_qcc_warm_qid" -q " + SELECT count() FROM ${EMPTY_TABLE} WHERE n = 3 SETTINGS ${FILTER_SETTINGS}") || fail "empty parquet warm select failed" +[[ $empty_qcc_warm == "0" ]] || fail "empty parquet warm: '${empty_qcc_warm}', expected 0" + +${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "flush logs before empty parquet hit check failed" +empty_qcc_hits=$(${CLICKHOUSE_CLIENT} -q " + SELECT ProfileEvents['QueryConditionCacheHits'] + FROM system.query_log + WHERE event_date >= yesterday() + AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND query_id = '${empty_qcc_warm_qid}' + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC + LIMIT 1 + FORMAT TabSeparated") || fail "empty parquet warm query_log failed" +[[ $empty_qcc_hits -ge 1 ]] || fail "empty parquet warm was not a condition-cache hit (QueryConditionCacheHits=${empty_qcc_hits})" + +curl -fsS -X PUT "$EMPTY_PARQUET_URL" --data-binary '' >/dev/null || fail "empty parquet replace put failed" + +empty_qcc_replaced_qid="idcache_skip_qcc_empty_replaced_${CLICKHOUSE_DATABASE}" +empty_qcc_replaced=$(${CLICKHOUSE_CLIENT} --query_id "$empty_qcc_replaced_qid" -q " + SELECT count() FROM ${EMPTY_TABLE} WHERE n = 3 SETTINGS ${FILTER_SETTINGS}") || fail "empty parquet replaced select failed" +[[ $empty_qcc_replaced == "0" ]] || fail "empty parquet replaced: '${empty_qcc_replaced}', expected 0" + +${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "flush logs after empty parquet replace failed" +empty_qcc_heads=$(${CLICKHOUSE_CLIENT} -q " + SELECT ProfileEvents['S3HeadObject'] + FROM system.query_log + WHERE event_date >= yesterday() + AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND query_id = '${empty_qcc_replaced_qid}' + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC + LIMIT 1 + FORMAT TabSeparated") || fail "empty parquet replaced query_log failed" +[[ $empty_qcc_heads -ge 1 ]] || fail "empty parquet replaced did not HEAD (S3HeadObject=${empty_qcc_heads})" + +${CLICKHOUSE_CLIENT} -q "DROP TABLE ${EMPTY_TABLE}" || fail "drop empty parquet table at end failed" + +echo OK diff --git a/tests/queries/0_stateless/05499_object_storage_identity_cache_unvalidated_replace.reference b/tests/queries/0_stateless/05499_object_storage_identity_cache_unvalidated_replace.reference new file mode 100644 index 0000000000000..d86bac9de59ab --- /dev/null +++ b/tests/queries/0_stateless/05499_object_storage_identity_cache_unvalidated_replace.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/05499_object_storage_identity_cache_unvalidated_replace.sh b/tests/queries/0_stateless/05499_object_storage_identity_cache_unvalidated_replace.sh new file mode 100755 index 0000000000000..c19a1e5c4c340 --- /dev/null +++ b/tests/queries/0_stateless/05499_object_storage_identity_cache_unvalidated_replace.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-parallel-replicas +# Tag no-fasttest: reads a file through the mock S3 server at localhost:11111 +# Tag no-parallel: the identity cache is process-global, and another test may clear it +# Tag no-parallel-replicas: per-query S3 request counts change when a read is spread across replicas + +# `s3_validate_etag_on_read = 0` sends the GET with no If-Match. A cached size must not stay the read +# bound: a small file is prefetched up to that size, so a larger external replace would be returned as +# the old prefix. The read HEADs again and returns the new rows. The HEAD also refreshes the entry, so +# the next read with validation on is a hit. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +URL="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" +COMMON_SETTINGS="max_threads = 1, use_hive_partitioning = 0, parallel_replicas_for_cluster_engines = 0, enable_filesystem_cache = 0, use_page_cache_for_object_storage = 0, use_object_storage_list_objects_cache = 0" +VALIDATED_SETTINGS="${COMMON_SETTINGS}, s3_validate_etag_on_read = 1" +UNVALIDATED_SETTINGS="${COMMON_SETTINGS}, s3_validate_etag_on_read = 0" + +fail() +{ + echo "FAIL: $*" >&2 + exit 1 +} + +# $1 label, $2 settings, $3 expected rows, $4 head mode (eq0 or ge1) +read_and_check() +{ + local label=$1 + local settings=$2 + local expected=$3 + local head_mode=$4 + local qid="idcache_unvalidated_${label}_${CLICKHOUSE_DATABASE}" + local data row head + + data=$(${CLICKHOUSE_CLIENT} --query_id "$qid" -q \ + "SELECT n FROM s3('${URL}', 'test', 'testtest', 'TSV', 'n UInt64') ORDER BY n SETTINGS ${settings} FORMAT TabSeparated") \ + || fail "${label}: select failed" + [[ $data == "$expected" ]] || fail "${label}: data '${data}', expected '${expected}'" + + ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "${label}: flush logs failed" + row=$(${CLICKHOUSE_CLIENT} -q " + SELECT ProfileEvents['S3HeadObject'] + FROM system.query_log + WHERE event_date >= yesterday() + AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND query_id = '${qid}' + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC + LIMIT 1 + FORMAT TabSeparated") || fail "${label}: query_log failed" + [[ -n $row ]] || fail "${label}: no query_log row" + head=$row + case "$head_mode" in + eq0) [[ $head == "0" ]] || fail "${label}: S3HeadObject=${head}, expected 0" ;; + ge1) [[ $head -ge 1 ]] || fail "${label}: S3HeadObject=${head}, expected >= 1" ;; + *) fail "${label}: bad mode ${head_mode}" ;; + esac +} + +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear failed" + +payload="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}.tsv" +printf '1\n' > "$payload" +curl -fsS -X PUT "$URL" --upload-file "$payload" >/dev/null || fail "first put failed" + +read_and_check cold "$VALIDATED_SETTINGS" $'1' ge1 +read_and_check warm "$VALIDATED_SETTINGS" $'1' eq0 + +# This PUT does not go through ClickHouse, so the cached identity still says the object is one row. +printf '10\n20\n30\n' > "$payload" +curl -fsS -X PUT "$URL" --upload-file "$payload" >/dev/null || fail "replace put failed" + +read_and_check replaced "$UNVALIDATED_SETTINGS" $'10\n20\n30' ge1 +read_and_check refreshed "$VALIDATED_SETTINGS" $'10\n20\n30' eq0 + +echo OK diff --git a/tests/queries/0_stateless/05500_object_storage_identity_cache_backup_replace.reference b/tests/queries/0_stateless/05500_object_storage_identity_cache_backup_replace.reference new file mode 100644 index 0000000000000..d86bac9de59ab --- /dev/null +++ b/tests/queries/0_stateless/05500_object_storage_identity_cache_backup_replace.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/05500_object_storage_identity_cache_backup_replace.sh b/tests/queries/0_stateless/05500_object_storage_identity_cache_backup_replace.sh new file mode 100755 index 0000000000000..3af56a327f621 --- /dev/null +++ b/tests/queries/0_stateless/05500_object_storage_identity_cache_backup_replace.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-parallel-replicas +# Tag no-fasttest: reads a file through the mock S3 server at localhost:11111 +# Tag no-parallel: the identity cache is process-global, and another test may clear it +# Tag no-parallel-replicas: per-query S3 request counts change when a read is spread across replicas + +# A backup overwrites its objects through BackupWriterS3, not S3ObjectStorage. After the first backup is +# read, the identity cache holds that ETag. A second backup to the same URL must drop the entry, or the +# next read sends If-Match for the previous ETag and fails with S3_OBJECT_CHANGED_DURING_READ. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +BACKUP_URL="http://localhost:11111/test/${CLICKHOUSE_TEST_UNIQUE_NAME}/" +OBJECT_URL="${BACKUP_URL}.backup" +COMMON_SETTINGS="max_threads = 1, use_hive_partitioning = 0, parallel_replicas_for_cluster_engines = 0, enable_filesystem_cache = 0, use_page_cache_for_object_storage = 0, use_object_storage_list_objects_cache = 0, s3_validate_etag_on_read = 1" + +fail() +{ + echo "FAIL: $*" >&2 + exit 1 +} + +# $1 label, $2 head mode (eq0 or ge1) +read_backup_and_check() +{ + local label=$1 + local head_mode=$2 + local qid="idcache_backup_${label}_${CLICKHOUSE_DATABASE}" + local data row head + + data=$(${CLICKHOUSE_CLIENT} --query_id "$qid" -q \ + "SELECT count() FROM s3('${OBJECT_URL}', 'test', 'testtest', 'LineAsString', 's String') SETTINGS ${COMMON_SETTINGS} FORMAT TabSeparated") \ + || fail "${label}: select failed" + [[ $data -ge 1 ]] || fail "${label}: data '${data}', expected at least 1 row" + + ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" || fail "${label}: flush logs failed" + row=$(${CLICKHOUSE_CLIENT} -q " + SELECT ProfileEvents['S3HeadObject'] + FROM system.query_log + WHERE event_date >= yesterday() + AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND query_id = '${qid}' + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC + LIMIT 1 + FORMAT TabSeparated") || fail "${label}: query_log failed" + [[ -n $row ]] || fail "${label}: no query_log row" + head=$row + case "$head_mode" in + eq0) [[ $head == "0" ]] || fail "${label}: S3HeadObject=${head}, expected 0" ;; + ge1) [[ $head -ge 1 ]] || fail "${label}: S3HeadObject=${head}, expected >= 1" ;; + *) fail "${label}: bad mode ${head_mode}" ;; + esac +} + +${CLICKHOUSE_CLIENT} -q "SYSTEM CLEAR OBJECT STORAGE IDENTITY CACHE" || fail "clear failed" +${CLICKHOUSE_CLIENT} -q "CREATE TABLE t (n UInt64) ENGINE = Memory" || fail "create failed" +${CLICKHOUSE_CLIENT} -q "INSERT INTO t VALUES (1)" || fail "insert failed" +${CLICKHOUSE_CLIENT} -q "BACKUP TABLE t TO S3('${BACKUP_URL}', 'test', 'testtest')" >/dev/null || fail "first backup failed" + +read_backup_and_check cold ge1 +read_backup_and_check warm eq0 + +${CLICKHOUSE_CLIENT} -q "INSERT INTO t VALUES (2)" || fail "second insert failed" +${CLICKHOUSE_CLIENT} -q "BACKUP TABLE t TO S3('${BACKUP_URL}', 'test', 'testtest')" >/dev/null || fail "second backup failed" + +read_backup_and_check replaced ge1 + +echo OK