Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 81 additions & 1 deletion src/Databases/DataLake/GlueCatalog.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
#if USE_AWS_S3 && USE_AVRO

#include <aws/glue/GlueClient.h>
#include <aws/glue/GlueErrors.h>
#include <aws/glue/model/GetTablesRequest.h>
#include <aws/glue/model/GetTableRequest.h>
#include <aws/glue/model/GetDatabaseRequest.h>
Expand Down Expand Up @@ -1195,12 +1196,46 @@ bool GlueCatalog::createTable(
return true;
}

GlueCatalog::TableVersion GlueCatalog::getTableVersion(
const String & namespace_name,
const String & table_name,
const DB::ForwardedAuthTokenPtr & auth_token) const
{
auto client = getClient(auth_token);

Aws::Glue::Model::GetTableRequest request;
request.SetDatabaseName(namespace_name);
request.SetName(table_name);

Aws::Glue::Model::GetTableOutcome outcome;
{
ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogGetTable);
auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogGetTableMicroseconds);
outcome = client.client->GetTable(request);
}

/// Without the version the update would be an unconditional overwrite, so a failure must not be swallowed.
if (!outcome.IsSuccess())
throw DB::Exception(
DB::ErrorCodes::DATALAKE_DATABASE_ERROR,
"Exception calling GetTable for table {}.{}: {}",
namespace_name, table_name, outcome.GetError().GetMessage());

const auto & table = outcome.GetResult().GetTable();
TableVersion result;
result.version_id = table.GetVersionId();
if (auto it = table.GetParameters().find("metadata_location"); it != table.GetParameters().end())
result.metadata_location = it->second;
return result;
}

bool GlueCatalog::updateTableInGlue(
const String & namespace_name,
const String & table_name,
const String & new_metadata_path,
const DB::ForwardedAuthTokenPtr & auth_token,
const std::vector<Aws::Glue::Model::Column> & columns) const
const std::vector<Aws::Glue::Model::Column> & columns,
const String & version_id) const
{
auto client = getClient(auth_token);

Expand Down Expand Up @@ -1234,6 +1269,12 @@ bool GlueCatalog::updateTableInGlue(

request.SetTableInput(table_input);

/// Optimistic concurrency: when a VersionId is supplied, Glue rejects the
/// update if the table has been modified since that version was read. It is only sound together
/// with a check of `metadata_location` made on the same read (see `updateMetadataIfCurrent`).
if (!version_id.empty())
request.SetVersionId(version_id);

Aws::Glue::Model::UpdateTableOutcome response;

{
Expand All @@ -1243,16 +1284,55 @@ bool GlueCatalog::updateTableInGlue(
}

if (!response.IsSuccess())
{
const auto error_type = response.GetError().GetErrorType();
if (error_type == Aws::Glue::GlueErrors::CONCURRENT_MODIFICATION
|| error_type == Aws::Glue::GlueErrors::VERSION_MISMATCH)
{
LOG_WARNING(getLogger("GlueCatalog"),
"Glue UpdateTable for {}.{} rejected (version conflict): {}",
namespace_name, table_name, response.GetError().GetMessage());
return false;
}
throw DB::Exception(DB::ErrorCodes::DATALAKE_DATABASE_ERROR, "Can not update metadata in glue catalog {}", response.GetError().GetMessage());
}

return true;
}

bool GlueCatalog::updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr /*new_snapshot*/, const DB::ForwardedAuthTokenPtr & auth_token) const
{
/// The caller gave no base metadata, so there is nothing to compare with and the update is unconditional.
return updateTableInGlue(namespace_name, table_name, new_metadata_path, auth_token);
}

bool GlueCatalog::updateMetadataIfCurrent(
const String & namespace_name,
const String & table_name,
const String & new_metadata_path,
Poco::JSON::Object::Ptr /*new_snapshot*/,
const String & expected_metadata_location,
const DB::ForwardedAuthTokenPtr & auth_token) const
{
/// Same order as Iceberg Java's `GlueTableOperations::doCommit`: read the table, check that it still
/// points at the writer's base, and send `UpdateTable` with the `VersionId` of that same read. The
/// `VersionId` alone would not do, because a read taken after a concurrent commit already includes it.
///
/// A table registered by `Location` only has no `metadata_location` parameter, and `getTableMetadata` reports
/// the newest metadata file found in storage as its location. There is no pointer to compare then: a writer
/// that registered one since the base was read makes the parameter non-empty and different, which is rejected
/// below, and the `VersionId` guards against a write between this read and the update.
const auto current = getTableVersion(namespace_name, table_name, auth_token);
if (!current.metadata_location.empty() && current.metadata_location != expected_metadata_location)
{
LOG_WARNING(getLogger("GlueCatalog"),
"Glue table {}.{} points at {} while the commit is based on {}",
namespace_name, table_name, current.metadata_location, expected_metadata_location);
return false;
}
return updateTableInGlue(namespace_name, table_name, new_metadata_path, auth_token, {}, current.version_id);
}

bool GlueCatalog::updateSchema(
const String & namespace_name,
const String & table_name,
Expand Down
26 changes: 25 additions & 1 deletion src/Databases/DataLake/GlueCatalog.h
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,14 @@ class GlueCatalog final : public ICatalog, private DB::WithContext

bool updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot, const DB::ForwardedAuthTokenPtr & auth_token) const override;

bool updateMetadataIfCurrent(
const String & namespace_name,
const String & table_name,
const String & new_metadata_path,
Poco::JSON::Object::Ptr new_snapshot,
const String & expected_metadata_location,
const DB::ForwardedAuthTokenPtr & auth_token) const override;

bool updateSchema(
const String & namespace_name,
const String & table_name,
Expand Down Expand Up @@ -189,14 +197,30 @@ class GlueCatalog final : public ICatalog, private DB::WithContext
Poco::JSON::Object::Ptr getOrFetchMetadataObject(
const AuthenticatedClient & client, const String & metadata_uri, const TableMetadata & table_metadata) const;

struct TableVersion
{
String version_id;
/// The raw `metadata_location` table parameter; empty if the table has none.
String metadata_location;
};

/// Reads the table with one `GetTable` call. Throws if the call fails.
TableVersion getTableVersion(
const String & namespace_name,
const String & table_name,
const DB::ForwardedAuthTokenPtr & auth_token) const;

/// Shared implementation for updateMetadata / updateSchema that optionally
/// sets StorageDescriptor columns in the Glue UpdateTable call.
/// When `version_id` is non-empty, the Glue UpdateTable request includes it
/// for optimistic concurrency control; a version mismatch returns false.
bool updateTableInGlue(
const String & namespace_name,
const String & table_name,
const String & new_metadata_path,
const DB::ForwardedAuthTokenPtr & auth_token,
const std::vector<Aws::Glue::Model::Column> & columns = {}) const;
const std::vector<Aws::Glue::Model::Column> & columns = {},
const String & version_id = {}) const;

mutable DB::CacheBase<String, Poco::JSON::Object::Ptr> metadata_objects;
mutable DB::CacheBase<String, FieldTypes, std::hash<String>, FieldTypesWeightFunction> current_schema_field_types;
Expand Down
11 changes: 11 additions & 0 deletions src/Databases/DataLake/ICatalog.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -523,6 +523,17 @@ bool ICatalog::updateMetadata(const String & /*namespace_name*/, const String &
throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "updateMetadata is not implemented");
}

bool ICatalog::updateMetadataIfCurrent(
const String & namespace_name,
const String & table_name,
const String & new_metadata_path,
Poco::JSON::Object::Ptr new_snapshot,
const String & /*expected_metadata_location*/,
const DB::ForwardedAuthTokenPtr & auth_token) const
{
return updateMetadata(namespace_name, table_name, new_metadata_path, new_snapshot, auth_token);
}

bool ICatalog::updateSchema(
const String & /*namespace_name*/,
const String & /*table_name*/,
Expand Down
13 changes: 13 additions & 0 deletions src/Databases/DataLake/ICatalog.h
Original file line number Diff line number Diff line change
Expand Up @@ -333,6 +333,19 @@ class ICatalog
/// Updates metadata in catalog.
virtual bool updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot, const DB::ForwardedAuthTokenPtr & auth_token) const;

/// Like `updateMetadata`, but the update must only take effect if the catalog still points at
/// `expected_metadata_location`, the metadata file the new snapshot was built on. It is the value the
/// catalog reported in `DataLakeSpecificProperties::iceberg_metadata_file_location`, unmodified.
/// Returns false if the catalog moved on in the meantime. A catalog that is itself a compare-and-swap
/// (REST) enforces the base through the update requirements and uses this default, which ignores it.
virtual bool updateMetadataIfCurrent(
const String & namespace_name,
const String & table_name,
const String & new_metadata_path,
Poco::JSON::Object::Ptr new_snapshot,
const String & expected_metadata_location,
const DB::ForwardedAuthTokenPtr & auth_token) const;

/// Commit a schema evolution (ADD/DROP/MODIFY/RENAME COLUMN) to the catalog.
/// `new_metadata_path` is the path of the freshly written `vN.metadata.json`; it is used by
/// non-transactional catalogs (e.g. Glue) that only store a pointer to the metadata file.
Expand Down
53 changes: 41 additions & 12 deletions src/Storages/ObjectStorage/DataLakes/Iceberg/BinPackRewrite.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
#include <Storages/ObjectStorage/Utils.h>
#include <Poco/JSON/Stringifier.h>
#include <Poco/String.h>
#include <Common/FailPoint.h>
#include <Common/FieldVisitorDump.h>
#include <Common/Logger.h>
#include <boost/container_hash/hash.hpp>
Expand All @@ -44,6 +45,11 @@ namespace DB::ErrorCodes
extern const int UNFINISHED;
}

namespace DB::FailPoints
{
extern const char iceberg_compaction_pause_before_metadata_commit[];
}

namespace DB::Setting
{
extern const SettingsUInt64 iceberg_target_data_file_size_bytes;
Expand Down Expand Up @@ -602,11 +608,16 @@ struct LatestMetadata
Poco::JSON::Object::Ptr object;
/// The catalog's metadata location the compaction is based on; empty without a catalog.
String catalog_metadata_location;
/// The same location exactly as the catalog reports it, which is what `updateMetadataIfCurrent` compares.
String raw_catalog_metadata_location;
};

String getCatalogMetadataLocation(const DataLake::ICatalog & catalog, const StorageID & storage_id, ContextPtr context)
/// The metadata location exactly as the catalog reports it. `table_metadata` receives the table metadata, which
/// `getMetadataLocation` needs to turn the raw value into a storage-relative one.
String getRawCatalogMetadataLocation(
const DataLake::ICatalog & catalog, const StorageID & storage_id, ContextPtr context, DataLake::TableMetadata & table_metadata)
{
DataLake::TableMetadata table_metadata = DataLake::TableMetadata().withDataLakeSpecificProperties().withLocation();
table_metadata = DataLake::TableMetadata().withDataLakeSpecificProperties().withLocation();
const auto & [namespace_name, table_name] = DataLake::parseTableName(storage_id.getTableName());
catalog.getTableMetadata(namespace_name, table_name, context, table_metadata);

Expand All @@ -617,7 +628,14 @@ String getCatalogMetadataLocation(const DataLake::ICatalog & catalog, const Stor
"Catalog did not return a metadata file location for table '{}.{}'",
namespace_name, table_name);

return table_metadata.getMetadataLocation(specific_properties->iceberg_metadata_file_location);
return specific_properties->iceberg_metadata_file_location;
}

String getCatalogMetadataLocation(const DataLake::ICatalog & catalog, const StorageID & storage_id, ContextPtr context)
{
DataLake::TableMetadata table_metadata;
const auto raw_location = getRawCatalogMetadataLocation(catalog, storage_id, context, table_metadata);
return table_metadata.getMetadataLocation(raw_location);
}

LatestMetadata readLatestMetadata(
Expand All @@ -633,7 +651,9 @@ LatestMetadata readLatestMetadata(
{
/// The catalog, not the newest file in the `metadata` directory, holds the current
/// snapshot: a newer file may be left by a commit that failed before updating the catalog.
String catalog_metadata_location = getCatalogMetadataLocation(*catalog, storage_id, context);
DataLake::TableMetadata table_metadata;
String raw_catalog_metadata_location = getRawCatalogMetadataLocation(*catalog, storage_id, context, table_metadata);
String catalog_metadata_location = table_metadata.getMetadataLocation(raw_catalog_metadata_location);

DataLakeStorageSettings effective_settings = data_lake_settings;
effective_settings[DataLakeStorageSetting::iceberg_metadata_file_path] = catalog_metadata_location;
Expand Down Expand Up @@ -662,7 +682,7 @@ LatestMetadata readLatestMetadata(
const auto max_directory_version = getMaxMetadataFileVersionInDirectory(object_storage, persistent_table_components.table_path);
const Int32 version = std::max(catalog_metadata.version, max_directory_version.value_or(catalog_metadata.version));

return {version, metadata_object, std::move(catalog_metadata_location)};
return {version, metadata_object, std::move(catalog_metadata_location), std::move(raw_catalog_metadata_location)};
}

const auto [metadata_version, metadata_file_path, _] = getLatestOrExplicitMetadataFileAndVersion(
Expand All @@ -686,7 +706,7 @@ LatestMetadata readLatestMetadata(
persistent_table_components.metadata_compression_method,
persistent_table_components.table_uuid);

return {metadata_version, metadata_object, {}};
return {metadata_version, metadata_object, {}, {}};
}

enum class MetadataFileOwner : uint8_t
Expand Down Expand Up @@ -1330,13 +1350,16 @@ BinPackCommitResult commitMetadata(
const std::shared_ptr<DataLake::ICatalog> & catalog,
const StorageID & storage_id,
const String & catalog_metadata_location,
const String & raw_catalog_metadata_location,
bool & keep_files_on_error,
const std::function<void()> & cleanup,
LoggerPtr log)
{
/// A catalog may repoint the table without a compare-and-swap (Glue `UpdateTable` is unconditional),
/// so a snapshot built on a superseded metadata file would replace the newer history. This check
/// narrows that window; it cannot close it.
/// A snapshot built on a superseded metadata file would replace the newer history. This check only
/// narrows that window, since a commit can still land after it. The window is closed by the catalog,
/// which `updateMetadataIfCurrent` asks to apply the update only if it still points at the base:
/// REST does it through the update requirements, Glue by comparing the location and sending
/// `UpdateTable` with the `VersionId` of that same read.
if (catalog)
{
const String current_location = getCatalogMetadataLocation(*catalog, storage_id, context);
Expand All @@ -1355,6 +1378,10 @@ BinPackCommitResult commitMetadata(
/// must not remove them. Paths that know the snapshot did not land call `cleanup` themselves.
keep_files_on_error = true;

/// Test hook for Altinity/ClickHouse#2733: suspend here so a test can land a concurrent commit
/// before the metadata file is written and the catalog pointer is updated.
FailPointInjection::pauseFailPoint(FailPoints::iceberg_compaction_pause_before_metadata_commit);

/// A transactional catalog writes the metadata file itself and is the source of truth for the
/// current metadata, so the storage-side metadata file and version hint are not written for it.
const bool catalog_writes_metadata_file = catalog && catalog->isTransactional();
Expand All @@ -1372,7 +1399,8 @@ BinPackCommitResult commitMetadata(
{
const auto catalog_filename = path_resolver.resolveForCatalog(generated_metadata_info.path);
const auto & [namespace_name, table_name] = DataLake::parseTableName(storage_id.getTableName());
if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, snapshot, context->getForwardedAuthToken()))
if (!catalog->updateMetadataIfCurrent(
namespace_name, table_name, catalog_filename, snapshot, raw_catalog_metadata_location, context->getForwardedAuthToken()))
{
/// A catalog reports both a lost race and a request whose outcome is unknown (a timeout or
/// a 5xx response) as a rejection, so the snapshot may still be live: the files stay.
Expand Down Expand Up @@ -1405,7 +1433,7 @@ BinPackCommitResult executeBinPackCompaction(
UInt64 target_size = settings[Setting::iceberg_target_data_file_size_bytes];
UInt64 min_size = settings[Setting::iceberg_min_data_file_size_bytes];

const auto [metadata_version, metadata_object, catalog_metadata_location]
const auto [metadata_version, metadata_object, catalog_metadata_location, raw_catalog_metadata_location]
= readLatestMetadata(persistent_table_components, object_storage, data_lake_settings, context, catalog, storage_id, log);

/// Format version 3 requires row lineage (`_row_id`, `_last_updated_sequence_number`) to be carried
Expand Down Expand Up @@ -1530,7 +1558,8 @@ BinPackCommitResult executeBinPackCompaction(
auto commit = commitMetadata(
metadata_object, snapshot_result.snapshot, generated_metadata_info,
generator, path_resolver, persistent_table_components, object_storage,
context, data_lake_settings, catalog, storage_id, catalog_metadata_location, keep_files_on_error, cleanup, log);
context, data_lake_settings, catalog, storage_id, catalog_metadata_location, raw_catalog_metadata_location,
keep_files_on_error, cleanup, log);

if (commit != BinPackCommitResult::Committed)
return commit;
Expand Down
Loading
Loading