diff --git a/src/Databases/DataLake/GlueCatalog.cpp b/src/Databases/DataLake/GlueCatalog.cpp index e97d585c1db31..0a3b4dc16334c 100644 --- a/src/Databases/DataLake/GlueCatalog.cpp +++ b/src/Databases/DataLake/GlueCatalog.cpp @@ -7,6 +7,7 @@ #if USE_AWS_S3 && USE_AVRO #include +#include #include #include #include @@ -1195,12 +1196,46 @@ bool GlueCatalog::createTable( return true; } +GlueCatalog::TableVersion GlueCatalog::getTableVersion( + const String & namespace_name, + const String & table_name, + const DB::ForwardedAuthTokenPtr & auth_token) const +{ + auto client = getClient(auth_token); + + Aws::Glue::Model::GetTableRequest request; + request.SetDatabaseName(namespace_name); + request.SetName(table_name); + + Aws::Glue::Model::GetTableOutcome outcome; + { + ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogGetTable); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogGetTableMicroseconds); + outcome = client.client->GetTable(request); + } + + /// Without the version the update would be an unconditional overwrite, so a failure must not be swallowed. + if (!outcome.IsSuccess()) + throw DB::Exception( + DB::ErrorCodes::DATALAKE_DATABASE_ERROR, + "Exception calling GetTable for table {}.{}: {}", + namespace_name, table_name, outcome.GetError().GetMessage()); + + const auto & table = outcome.GetResult().GetTable(); + TableVersion result; + result.version_id = table.GetVersionId(); + if (auto it = table.GetParameters().find("metadata_location"); it != table.GetParameters().end()) + result.metadata_location = it->second; + return result; +} + bool GlueCatalog::updateTableInGlue( const String & namespace_name, const String & table_name, const String & new_metadata_path, const DB::ForwardedAuthTokenPtr & auth_token, - const std::vector & columns) const + const std::vector & columns, + const String & version_id) const { auto client = getClient(auth_token); @@ -1234,6 +1269,12 @@ bool GlueCatalog::updateTableInGlue( request.SetTableInput(table_input); + /// Optimistic concurrency: when a VersionId is supplied, Glue rejects the + /// update if the table has been modified since that version was read. It is only sound together + /// with a check of `metadata_location` made on the same read (see `updateMetadataIfCurrent`). + if (!version_id.empty()) + request.SetVersionId(version_id); + Aws::Glue::Model::UpdateTableOutcome response; { @@ -1243,16 +1284,55 @@ bool GlueCatalog::updateTableInGlue( } if (!response.IsSuccess()) + { + const auto error_type = response.GetError().GetErrorType(); + if (error_type == Aws::Glue::GlueErrors::CONCURRENT_MODIFICATION + || error_type == Aws::Glue::GlueErrors::VERSION_MISMATCH) + { + LOG_WARNING(getLogger("GlueCatalog"), + "Glue UpdateTable for {}.{} rejected (version conflict): {}", + namespace_name, table_name, response.GetError().GetMessage()); + return false; + } throw DB::Exception(DB::ErrorCodes::DATALAKE_DATABASE_ERROR, "Can not update metadata in glue catalog {}", response.GetError().GetMessage()); + } return true; } bool GlueCatalog::updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr /*new_snapshot*/, const DB::ForwardedAuthTokenPtr & auth_token) const { + /// The caller gave no base metadata, so there is nothing to compare with and the update is unconditional. return updateTableInGlue(namespace_name, table_name, new_metadata_path, auth_token); } +bool GlueCatalog::updateMetadataIfCurrent( + const String & namespace_name, + const String & table_name, + const String & new_metadata_path, + Poco::JSON::Object::Ptr /*new_snapshot*/, + const String & expected_metadata_location, + const DB::ForwardedAuthTokenPtr & auth_token) const +{ + /// Same order as Iceberg Java's `GlueTableOperations::doCommit`: read the table, check that it still + /// points at the writer's base, and send `UpdateTable` with the `VersionId` of that same read. The + /// `VersionId` alone would not do, because a read taken after a concurrent commit already includes it. + /// + /// A table registered by `Location` only has no `metadata_location` parameter, and `getTableMetadata` reports + /// the newest metadata file found in storage as its location. There is no pointer to compare then: a writer + /// that registered one since the base was read makes the parameter non-empty and different, which is rejected + /// below, and the `VersionId` guards against a write between this read and the update. + const auto current = getTableVersion(namespace_name, table_name, auth_token); + if (!current.metadata_location.empty() && current.metadata_location != expected_metadata_location) + { + LOG_WARNING(getLogger("GlueCatalog"), + "Glue table {}.{} points at {} while the commit is based on {}", + namespace_name, table_name, current.metadata_location, expected_metadata_location); + return false; + } + return updateTableInGlue(namespace_name, table_name, new_metadata_path, auth_token, {}, current.version_id); +} + bool GlueCatalog::updateSchema( const String & namespace_name, const String & table_name, diff --git a/src/Databases/DataLake/GlueCatalog.h b/src/Databases/DataLake/GlueCatalog.h index 0dd11e7b199a7..4d07981c257e4 100644 --- a/src/Databases/DataLake/GlueCatalog.h +++ b/src/Databases/DataLake/GlueCatalog.h @@ -87,6 +87,14 @@ class GlueCatalog final : public ICatalog, private DB::WithContext bool updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot, const DB::ForwardedAuthTokenPtr & auth_token) const override; + bool updateMetadataIfCurrent( + const String & namespace_name, + const String & table_name, + const String & new_metadata_path, + Poco::JSON::Object::Ptr new_snapshot, + const String & expected_metadata_location, + const DB::ForwardedAuthTokenPtr & auth_token) const override; + bool updateSchema( const String & namespace_name, const String & table_name, @@ -189,14 +197,30 @@ class GlueCatalog final : public ICatalog, private DB::WithContext Poco::JSON::Object::Ptr getOrFetchMetadataObject( const AuthenticatedClient & client, const String & metadata_uri, const TableMetadata & table_metadata) const; + struct TableVersion + { + String version_id; + /// The raw `metadata_location` table parameter; empty if the table has none. + String metadata_location; + }; + + /// Reads the table with one `GetTable` call. Throws if the call fails. + TableVersion getTableVersion( + const String & namespace_name, + const String & table_name, + const DB::ForwardedAuthTokenPtr & auth_token) const; + /// Shared implementation for updateMetadata / updateSchema that optionally /// sets StorageDescriptor columns in the Glue UpdateTable call. + /// When `version_id` is non-empty, the Glue UpdateTable request includes it + /// for optimistic concurrency control; a version mismatch returns false. bool updateTableInGlue( const String & namespace_name, const String & table_name, const String & new_metadata_path, const DB::ForwardedAuthTokenPtr & auth_token, - const std::vector & columns = {}) const; + const std::vector & columns = {}, + const String & version_id = {}) const; mutable DB::CacheBase metadata_objects; mutable DB::CacheBase, FieldTypesWeightFunction> current_schema_field_types; diff --git a/src/Databases/DataLake/ICatalog.cpp b/src/Databases/DataLake/ICatalog.cpp index 9cc28afc48917..9f937d371f224 100644 --- a/src/Databases/DataLake/ICatalog.cpp +++ b/src/Databases/DataLake/ICatalog.cpp @@ -523,6 +523,17 @@ bool ICatalog::updateMetadata(const String & /*namespace_name*/, const String & throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "updateMetadata is not implemented"); } +bool ICatalog::updateMetadataIfCurrent( + const String & namespace_name, + const String & table_name, + const String & new_metadata_path, + Poco::JSON::Object::Ptr new_snapshot, + const String & /*expected_metadata_location*/, + const DB::ForwardedAuthTokenPtr & auth_token) const +{ + return updateMetadata(namespace_name, table_name, new_metadata_path, new_snapshot, auth_token); +} + bool ICatalog::updateSchema( const String & /*namespace_name*/, const String & /*table_name*/, diff --git a/src/Databases/DataLake/ICatalog.h b/src/Databases/DataLake/ICatalog.h index 5e092e92b75f2..a3f3a2938b728 100644 --- a/src/Databases/DataLake/ICatalog.h +++ b/src/Databases/DataLake/ICatalog.h @@ -333,6 +333,19 @@ class ICatalog /// Updates metadata in catalog. virtual bool updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot, const DB::ForwardedAuthTokenPtr & auth_token) const; + /// Like `updateMetadata`, but the update must only take effect if the catalog still points at + /// `expected_metadata_location`, the metadata file the new snapshot was built on. It is the value the + /// catalog reported in `DataLakeSpecificProperties::iceberg_metadata_file_location`, unmodified. + /// Returns false if the catalog moved on in the meantime. A catalog that is itself a compare-and-swap + /// (REST) enforces the base through the update requirements and uses this default, which ignores it. + virtual bool updateMetadataIfCurrent( + const String & namespace_name, + const String & table_name, + const String & new_metadata_path, + Poco::JSON::Object::Ptr new_snapshot, + const String & expected_metadata_location, + const DB::ForwardedAuthTokenPtr & auth_token) const; + /// Commit a schema evolution (ADD/DROP/MODIFY/RENAME COLUMN) to the catalog. /// `new_metadata_path` is the path of the freshly written `vN.metadata.json`; it is used by /// non-transactional catalogs (e.g. Glue) that only store a pointer to the metadata file. diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/BinPackRewrite.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/BinPackRewrite.cpp index aec7e2be05ffd..d5f7c2d9e5be2 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/BinPackRewrite.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/BinPackRewrite.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -44,6 +45,11 @@ namespace DB::ErrorCodes extern const int UNFINISHED; } +namespace DB::FailPoints +{ + extern const char iceberg_compaction_pause_before_metadata_commit[]; +} + namespace DB::Setting { extern const SettingsUInt64 iceberg_target_data_file_size_bytes; @@ -602,11 +608,16 @@ struct LatestMetadata Poco::JSON::Object::Ptr object; /// The catalog's metadata location the compaction is based on; empty without a catalog. String catalog_metadata_location; + /// The same location exactly as the catalog reports it, which is what `updateMetadataIfCurrent` compares. + String raw_catalog_metadata_location; }; -String getCatalogMetadataLocation(const DataLake::ICatalog & catalog, const StorageID & storage_id, ContextPtr context) +/// The metadata location exactly as the catalog reports it. `table_metadata` receives the table metadata, which +/// `getMetadataLocation` needs to turn the raw value into a storage-relative one. +String getRawCatalogMetadataLocation( + const DataLake::ICatalog & catalog, const StorageID & storage_id, ContextPtr context, DataLake::TableMetadata & table_metadata) { - DataLake::TableMetadata table_metadata = DataLake::TableMetadata().withDataLakeSpecificProperties().withLocation(); + table_metadata = DataLake::TableMetadata().withDataLakeSpecificProperties().withLocation(); const auto & [namespace_name, table_name] = DataLake::parseTableName(storage_id.getTableName()); catalog.getTableMetadata(namespace_name, table_name, context, table_metadata); @@ -617,7 +628,14 @@ String getCatalogMetadataLocation(const DataLake::ICatalog & catalog, const Stor "Catalog did not return a metadata file location for table '{}.{}'", namespace_name, table_name); - return table_metadata.getMetadataLocation(specific_properties->iceberg_metadata_file_location); + return specific_properties->iceberg_metadata_file_location; +} + +String getCatalogMetadataLocation(const DataLake::ICatalog & catalog, const StorageID & storage_id, ContextPtr context) +{ + DataLake::TableMetadata table_metadata; + const auto raw_location = getRawCatalogMetadataLocation(catalog, storage_id, context, table_metadata); + return table_metadata.getMetadataLocation(raw_location); } LatestMetadata readLatestMetadata( @@ -633,7 +651,9 @@ LatestMetadata readLatestMetadata( { /// The catalog, not the newest file in the `metadata` directory, holds the current /// snapshot: a newer file may be left by a commit that failed before updating the catalog. - String catalog_metadata_location = getCatalogMetadataLocation(*catalog, storage_id, context); + DataLake::TableMetadata table_metadata; + String raw_catalog_metadata_location = getRawCatalogMetadataLocation(*catalog, storage_id, context, table_metadata); + String catalog_metadata_location = table_metadata.getMetadataLocation(raw_catalog_metadata_location); DataLakeStorageSettings effective_settings = data_lake_settings; effective_settings[DataLakeStorageSetting::iceberg_metadata_file_path] = catalog_metadata_location; @@ -662,7 +682,7 @@ LatestMetadata readLatestMetadata( const auto max_directory_version = getMaxMetadataFileVersionInDirectory(object_storage, persistent_table_components.table_path); const Int32 version = std::max(catalog_metadata.version, max_directory_version.value_or(catalog_metadata.version)); - return {version, metadata_object, std::move(catalog_metadata_location)}; + return {version, metadata_object, std::move(catalog_metadata_location), std::move(raw_catalog_metadata_location)}; } const auto [metadata_version, metadata_file_path, _] = getLatestOrExplicitMetadataFileAndVersion( @@ -686,7 +706,7 @@ LatestMetadata readLatestMetadata( persistent_table_components.metadata_compression_method, persistent_table_components.table_uuid); - return {metadata_version, metadata_object, {}}; + return {metadata_version, metadata_object, {}, {}}; } enum class MetadataFileOwner : uint8_t @@ -1330,13 +1350,16 @@ BinPackCommitResult commitMetadata( const std::shared_ptr & catalog, const StorageID & storage_id, const String & catalog_metadata_location, + const String & raw_catalog_metadata_location, bool & keep_files_on_error, const std::function & cleanup, LoggerPtr log) { - /// A catalog may repoint the table without a compare-and-swap (Glue `UpdateTable` is unconditional), - /// so a snapshot built on a superseded metadata file would replace the newer history. This check - /// narrows that window; it cannot close it. + /// A snapshot built on a superseded metadata file would replace the newer history. This check only + /// narrows that window, since a commit can still land after it. The window is closed by the catalog, + /// which `updateMetadataIfCurrent` asks to apply the update only if it still points at the base: + /// REST does it through the update requirements, Glue by comparing the location and sending + /// `UpdateTable` with the `VersionId` of that same read. if (catalog) { const String current_location = getCatalogMetadataLocation(*catalog, storage_id, context); @@ -1355,6 +1378,10 @@ BinPackCommitResult commitMetadata( /// must not remove them. Paths that know the snapshot did not land call `cleanup` themselves. keep_files_on_error = true; + /// Test hook for Altinity/ClickHouse#2733: suspend here so a test can land a concurrent commit + /// before the metadata file is written and the catalog pointer is updated. + FailPointInjection::pauseFailPoint(FailPoints::iceberg_compaction_pause_before_metadata_commit); + /// A transactional catalog writes the metadata file itself and is the source of truth for the /// current metadata, so the storage-side metadata file and version hint are not written for it. const bool catalog_writes_metadata_file = catalog && catalog->isTransactional(); @@ -1372,7 +1399,8 @@ BinPackCommitResult commitMetadata( { const auto catalog_filename = path_resolver.resolveForCatalog(generated_metadata_info.path); const auto & [namespace_name, table_name] = DataLake::parseTableName(storage_id.getTableName()); - if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, snapshot, context->getForwardedAuthToken())) + if (!catalog->updateMetadataIfCurrent( + namespace_name, table_name, catalog_filename, snapshot, raw_catalog_metadata_location, context->getForwardedAuthToken())) { /// A catalog reports both a lost race and a request whose outcome is unknown (a timeout or /// a 5xx response) as a rejection, so the snapshot may still be live: the files stay. @@ -1405,7 +1433,7 @@ BinPackCommitResult executeBinPackCompaction( UInt64 target_size = settings[Setting::iceberg_target_data_file_size_bytes]; UInt64 min_size = settings[Setting::iceberg_min_data_file_size_bytes]; - const auto [metadata_version, metadata_object, catalog_metadata_location] + const auto [metadata_version, metadata_object, catalog_metadata_location, raw_catalog_metadata_location] = readLatestMetadata(persistent_table_components, object_storage, data_lake_settings, context, catalog, storage_id, log); /// Format version 3 requires row lineage (`_row_id`, `_last_updated_sequence_number`) to be carried @@ -1530,7 +1558,8 @@ BinPackCommitResult executeBinPackCompaction( auto commit = commitMetadata( metadata_object, snapshot_result.snapshot, generated_metadata_info, generator, path_resolver, persistent_table_components, object_storage, - context, data_lake_settings, catalog, storage_id, catalog_metadata_location, keep_files_on_error, cleanup, log); + context, data_lake_settings, catalog, storage_id, catalog_metadata_location, raw_catalog_metadata_location, + keep_files_on_error, cleanup, log); if (commit != BinPackCommitResult::Committed) return commit; diff --git a/tests/integration/test_database_glue/test.py b/tests/integration/test_database_glue/test.py index f5d4d2027619d..9e9f43c426fe7 100644 --- a/tests/integration/test_database_glue/test.py +++ b/tests/integration/test_database_glue/test.py @@ -2251,3 +2251,127 @@ def test_catalog_schema_with_empty_column_name_is_rejected(started_cluster): assert node.query("SELECT 1") == "1\n" node.query(f"DROP DATABASE IF EXISTS {db_name} SYNC") + + +def test_optimize_bin_pack_concurrent_commit(started_cluster): + """Regression test for https://github.com/Altinity/ClickHouse/issues/2733. + + Bin-pack OPTIMIZE on a Glue-catalog table pauses right before it publishes its metadata. While it is + paused, pyiceberg lands another commit. pyiceberg names its metadata files differently from ClickHouse, + so the commit does not collide with OPTIMIZE's metadata file in storage and only the catalog can notice + it. When OPTIMIZE resumes, Glue must refuse its update, because the table no longer points at the + metadata OPTIMIZE was based on, and OPTIMIZE must retry on the new snapshot. No row may be lost.""" + node = started_cluster.instances["node1"] + + test_ref = f"test_optimize_race_{uuid.uuid4()}" + table_name = f"{test_ref}_table" + root_namespace = f"{test_ref}_namespace" + table_ref = f"{CATALOG_NAME}.`{root_namespace}.{table_name}`" + + catalog = load_catalog_impl(started_cluster) + catalog.create_namespace(root_namespace) + table = create_table(catalog, root_namespace, table_name, DEFAULT_SCHEMA, PartitionSpec(), DEFAULT_SORT_ORDER, table_name) + + # Two separate appends produce two small data files, so bin-pack has work. + table.append(generate_arrow_data(2)) + table.append(generate_arrow_data(3)) + + create_clickhouse_glue_database(started_cluster, node, CATALOG_NAME) + assert node.query(f"SELECT count() FROM {table_ref}").strip() == "5" + + node.query("SYSTEM ENABLE FAILPOINT iceberg_compaction_pause_before_metadata_commit") + + executor = ThreadPoolExecutor(max_workers=2) + try: + # 1. Wait for the compaction to pause at the failpoint. + wait_future = executor.submit( + lambda: node.query( + "SYSTEM WAIT FAILPOINT iceberg_compaction_pause_before_metadata_commit PAUSE", + timeout=120, + ) + ) + + # 2. Start OPTIMIZE; it rewrites the two small files, then pauses before it publishes. + optimize_future = executor.submit( + lambda: node.query( + f"OPTIMIZE TABLE {table_ref}", + settings={ + "allow_experimental_iceberg_compaction": 1, + "iceberg_target_data_file_size_bytes": 10 * 1024 * 1024, + "iceberg_min_data_file_size_bytes": 10 * 1024 * 1024, + }, + timeout=180, + ) + ) + + # Wait for the pause to actually trigger before the concurrent commit. + wait_future.result(timeout=120) + + # 3. Another writer commits in the publish window. + catalog.load_table(f"{root_namespace}.{table_name}").append(generate_arrow_data(4)) + + # 4. Release the failpoint; OPTIMIZE's publish must be rejected and retried on the new snapshot. + node.query("SYSTEM DISABLE FAILPOINT iceberg_compaction_pause_before_metadata_commit") + optimize_future.result(timeout=180) + finally: + # Always disable the failpoint, otherwise a failure above leaves it enabled for later tests + # on the same instance. The disable is idempotent, so no exception needs to be swallowed. + node.query("SYSTEM DISABLE FAILPOINT iceberg_compaction_pause_before_metadata_commit") + executor.shutdown(wait=False) + + # 5. The rows committed during the publish window must remain in the current snapshot. + assert catalog.load_table(f"{root_namespace}.{table_name}").scan().to_arrow().num_rows == 9 + assert node.query(f"SELECT count() FROM {table_ref}").strip() == "9" + + +def test_optimize_bin_pack_table_without_metadata_location(started_cluster): + """A table registered in Glue by `Location` only has no `metadata_location` parameter, and ClickHouse + reads its current metadata from storage. `OPTIMIZE` must still be able to compact it and register the pointer.""" + node = started_cluster.instances["node1"] + + test_ref = f"test_optimize_no_metadata_location_{uuid.uuid4()}" + table_name = f"{test_ref}_table" + root_namespace = f"{test_ref}_namespace" + table_ref = f"{CATALOG_NAME}.`{root_namespace}.{table_name}`" + + catalog = load_catalog_impl(started_cluster) + catalog.create_namespace(root_namespace) + table = create_table(catalog, root_namespace, table_name, DEFAULT_SCHEMA, PartitionSpec(), DEFAULT_SORT_ORDER, table_name) + + # Two separate appends produce two small data files, so bin-pack has work. + table.append(generate_arrow_data(2)) + table.append(generate_arrow_data(3)) + + glue_client = boto3.client( + "glue", region_name="us-east-1", endpoint_url=get_glue_local_url(started_cluster) + ) + table_info = glue_client.get_table(DatabaseName=root_namespace, Name=table_name)["Table"] + assert "metadata_location" in table_info["Parameters"], "Test setup failed: metadata_location should exist initially" + glue_client.update_table( + DatabaseName=root_namespace, + TableInput={ + "Name": table_name, + "StorageDescriptor": table_info["StorageDescriptor"], + "Parameters": {k: v for k, v in table_info["Parameters"].items() if k != "metadata_location"}, + "TableType": table_info.get("TableType", "EXTERNAL_TABLE"), + }, + ) + table_info = glue_client.get_table(DatabaseName=root_namespace, Name=table_name)["Table"] + assert "metadata_location" not in table_info["Parameters"], "Failed to remove metadata_location" + + create_clickhouse_glue_database(started_cluster, node, CATALOG_NAME) + assert node.query(f"SELECT count() FROM {table_ref}").strip() == "5" + + node.query( + f"OPTIMIZE TABLE {table_ref}", + settings={ + "allow_experimental_iceberg_compaction": 1, + "iceberg_target_data_file_size_bytes": 10 * 1024 * 1024, + "iceberg_min_data_file_size_bytes": 10 * 1024 * 1024, + }, + ) + + # The update registered the pointer, and no row was lost. + table_info = glue_client.get_table(DatabaseName=root_namespace, Name=table_name)["Table"] + assert table_info["Parameters"].get("metadata_location"), table_info["Parameters"] + assert node.query(f"SELECT count() FROM {table_ref}").strip() == "5"