Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/Common/FailPoint.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,9 @@ static struct InitFiu
PAUSEABLE_ONCE(iceberg_drop_partition_pause_after_discovery) \
ONCE(iceberg_alter_catalog_update_schema_fail) \
ONCE(iceberg_alter_catalog_commit_reported_as_failed) \
ONCE(iceberg_catalog_update_metadata_fail) \
ONCE(iceberg_catalog_update_metadata_reported_as_failed) \
ONCE(iceberg_glue_update_table_fail) \
REGULAR(storage_cluster_read_sleep) \
ONCE(iceberg_writes_non_retry_cleanup) \
ONCE(iceberg_writes_post_publish_throw) \
Expand Down
4 changes: 4 additions & 0 deletions src/Databases/DataLake/GlueCatalog.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ namespace DB::ErrorCodes
namespace DB::FailPoints
{
extern const char check_database_datalake_negative[];
extern const char iceberg_glue_update_table_fail[];
}

namespace DB::Setting
Expand Down Expand Up @@ -1234,6 +1235,9 @@ bool GlueCatalog::updateTableInGlue(

request.SetTableInput(table_input);

/// Simulates Glue rejecting the update, e.g. because of a `VersionId` mismatch with a concurrent commit.
fiu_do_on(DB::FailPoints::iceberg_glue_update_table_fail, { return false; });

Aws::Glue::Model::UpdateTableOutcome response;

{
Expand Down
10 changes: 10 additions & 0 deletions src/Databases/DataLake/RestCatalog.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,8 @@ namespace DB::FailPoints
extern const char check_database_datalake_negative[];
extern const char iceberg_alter_catalog_update_schema_fail[];
extern const char iceberg_alter_catalog_commit_reported_as_failed[];
extern const char iceberg_catalog_update_metadata_fail[];
extern const char iceberg_catalog_update_metadata_reported_as_failed[];
}

namespace ProfileEvents
Expand Down Expand Up @@ -2766,6 +2768,9 @@ bool RestCatalog::updateMetadata(const String & namespace_name, const String & t

auto request_body = buildUpdateMetadataRequestBody(namespace_name, table_name, new_snapshot);

/// Simulates a retryable failure where the catalog did not apply the update.
fiu_do_on(DB::FailPoints::iceberg_catalog_update_metadata_fail, { return false; });

try
{
ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUpdateTable);
Expand All @@ -2791,6 +2796,11 @@ bool RestCatalog::updateMetadata(const String & namespace_name, const String & t
table_name,
ex.displayText());
}

/// Simulates the Iceberg "commit state unknown" case: the catalog applied the update but the
/// client observes a failure, e.g. because a proxy turned the response into a 5xx.
fiu_do_on(DB::FailPoints::iceberg_catalog_update_metadata_reported_as_failed, { return false; });

return true;
}

Expand Down
132 changes: 124 additions & 8 deletions src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
#include <filesystem>
#include <limits>
#include <optional>
#include <string>
Expand All @@ -8,6 +9,7 @@
#include <Disks/DiskObjectStorage/ObjectStorages/StoredObject.h>
#include <Formats/FormatFactory.h>
#include <Databases/DataLake/Common.h>
#include <Databases/DataLake/ICatalog.h>
#include <Formats/FormatParserSharedResources.h>
#include <IO/CompressionMethod.h>
#include <Interpreters/FileCache/FileSegment.h>
Expand Down Expand Up @@ -42,6 +44,7 @@
namespace DB::ErrorCodes
{
extern const int BAD_ARGUMENTS;
extern const int DATALAKE_DATABASE_ERROR;
extern const int LOGICAL_ERROR;
extern const int CANNOT_UNLINK;
extern const int ICEBERG_SPECIFICATION_VIOLATION;
Expand Down Expand Up @@ -488,6 +491,53 @@ static void writeDataFiles(
}
}

/// Whether the table metadata that `catalog` currently points to contains a snapshot with `snapshot_id`.
/// Used to resolve the "commit state unknown" case, where the catalog may have applied an update
/// although the client observed a failure (e.g. a proxy turning the response into a 5xx).
/// Throws if the catalog or the metadata file cannot be read, so that the caller never treats
/// an unknown state as "not committed".
static bool isSnapshotInCatalogMetadata(
const std::shared_ptr<DataLake::ICatalog> & catalog,
const StorageID & table_id,
Int64 snapshot_id,
const PersistentTableComponents & persistent_table_components,
const DataLakeStorageSettings & data_lake_settings,
ObjectStoragePtr object_storage,
ContextPtr context,
LoggerPtr log)
{
const auto metadata_file_info = getLatestMetadataFileAndVersionWithCatalog(
object_storage,
catalog,
table_id.getTableName(),
persistent_table_components.table_path,
data_lake_settings,
persistent_table_components.metadata_cache,
context,
log.get(),
persistent_table_components.table_uuid,
persistent_table_components.metadata_compression_method);
auto metadata_object = getMetadataJSONObject(
metadata_file_info.path,
object_storage,
persistent_table_components.metadata_cache,
context,
log,
metadata_file_info.compression_method,
persistent_table_components.table_uuid);

if (!metadata_object->has(Iceberg::f_snapshots))
return false;

auto snapshots = metadata_object->get(Iceberg::f_snapshots).extract<Poco::JSON::Array::Ptr>();
for (size_t i = 0; i < snapshots->size(); ++i)
{
if (snapshots->getObject(static_cast<UInt32>(i))->getValue<Int64>(Iceberg::f_metadata_snapshot_id) == snapshot_id)
return true;
}
return false;
}

static bool writeConsolidatedManifestFile(
int metadata_version,
const String & metadata_file_path,
Expand Down Expand Up @@ -832,9 +882,21 @@ static bool writeConsolidatedManifestFile(
/// Parallel to consolidated_manifest_paths: each manifest's partition fields (value + type), used to recompute the manifest-list `partitions` summary.
std::vector<std::vector<std::pair<Field, DataTypePtr>>> entry_partition_summaries;

/// Whether the new snapshot may be visible to readers: through the metadata file and version hint
/// in storage, or through the catalog pointer. From then on the new manifests and manifest list may be
/// referenced by the current snapshot, so they must never be deleted: an orphaned file is harmless,
/// a deleted live one makes the table unreadable.
bool published = false;

/// Cleanup for both commit conflict and exceptions; paths are tracked before writeObject so partially-created objects are removed (removeObjectIfExists tolerates missing objects).
auto cleanup = [&]()
{
if (published)
{
LOG_INFO(log, "Not removing the new manifest files: the new snapshot may already be visible");
return;
}

for (const auto & mp : consolidated_manifest_paths)
{
try
Expand Down Expand Up @@ -1009,16 +1071,68 @@ static bool writeConsolidatedManifestFile(
return false;
}

/// The metadata file and version hint in storage now name the new snapshot.
if (!catalog_writes_metadata_file)
published = true;

/// Advance the catalog pointer to the new metadata so catalog-based readers see the compacted snapshot.
if (catalog)
{
auto catalog_filename = path_resolver.resolveForCatalog(generated_metadata_info.path);
const auto & [namespace_name, table_name] = DataLake::parseTableName(table_id.getTableName());
if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot.snapshot, context->getForwardedAuthToken()))

if (!catalog_writes_metadata_file)
{
LOG_INFO(log, "Metadata commit conflict detected via catalog, cleaning up temporary files");
cleanup();
return false;
/// The snapshot is already visible through storage, so a failed catalog update cannot be
/// undone by deleting files. Retrying is not correct either: the retry would read the
/// compacted metadata, find nothing to do and report success while the catalog pointer
/// still names the previous metadata. Keep the files and report the error.
if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot.snapshot, context->getForwardedAuthToken()))
throw Exception(
ErrorCodes::DATALAKE_DATABASE_ERROR,
"Compacted Iceberg metadata {} of table {}.{} was committed to storage, but the catalog "
"rejected the update of the table pointer. The table in storage is consistent; "
"the catalog still names the previous metadata",
catalog_filename, namespace_name, table_name);
}
else
{
bool updated = false;
std::exception_ptr update_exception;
try
{
updated = catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot.snapshot, context->getForwardedAuthToken());
}
catch (...)
{
update_exception = std::current_exception();
}

if (!updated)
{
/// Commit state unknown: the catalog may have applied the update although the client
/// saw a failure. Until the catalog proves otherwise, the new files may be live, so
/// keep them even if the check below throws.
published = true;
const Int64 new_snapshot_id = new_snapshot.snapshot->getValue<Int64>(Iceberg::f_metadata_snapshot_id);
if (isSnapshotInCatalogMetadata(
catalog, table_id, new_snapshot_id, persistent_table_components, data_lake_settings, object_storage, context, log))
{
LOG_INFO(log, "Catalog update for snapshot {} was reported as failed, but the catalog has applied it", new_snapshot_id);
}
else
{
/// The update did not land: the new files are not referenced by any snapshot.
published = false;
if (update_exception)
std::rethrow_exception(update_exception);

LOG_INFO(log, "Metadata commit conflict detected via catalog, cleaning up temporary files");
cleanup();
return false;
}
}
published = true;
}
}
}
Expand Down Expand Up @@ -1536,17 +1650,19 @@ void compactIcebergManifests(
if (attempt > 0)
LOG_INFO(log, "Retrying manifest compaction (attempt {}/{})", attempt + 1, MAX_COMPACTION_RETRIES);

const auto [metadata_version, metadata_file_path, _] = getLatestOrExplicitMetadataFileAndVersion(
/// With a catalog, the catalog names the current metadata file. Listing the storage instead may pick the
/// metadata of another table that shares the location, or miss a commit that is visible only in the catalog.
const auto [metadata_version, metadata_file_path, _] = getLatestMetadataFileAndVersionWithCatalog(
object_storage_,
catalog,
table_id.getTableName(),
persistent_table_components.table_path,
data_lake_settings,
persistent_table_components.metadata_cache,
context_,
log.get(),
persistent_table_components.table_uuid,
persistent_table_components.metadata_compression_method,
/* force_fetch_latest_metadata */ true,
/* ignore_explicit_metadata_file_path */ true);
persistent_table_components.metadata_compression_method);

auto metadata_object = getMetadataJSONObject(
metadata_file_path,
Expand Down
101 changes: 101 additions & 0 deletions tests/integration/test_database_glue/test.py
Original file line number Diff line number Diff line change
Expand Up @@ -2251,3 +2251,104 @@ def test_catalog_schema_with_empty_column_name_is_rejected(started_cluster):
assert node.query("SELECT 1") == "1\n"

node.query(f"DROP DATABASE IF EXISTS {db_name} SYNC")


def test_optimize_manifest_glue_update_fails(started_cluster):
"""`OPTIMIZE TABLE ... MANIFEST` must not delete the files of a snapshot that is visible in storage.

For Glue the compacted metadata file is written to storage before the catalog pointer is updated.
When Glue then rejects the update (e.g. a `VersionId` mismatch with a concurrent commit), the new
snapshot is already the latest one in storage, so its manifests must stay in place.
https://github.com/Altinity/ClickHouse/issues/2416
"""
import gzip
import io
import re

import avro.datafile
import avro.io

node = started_cluster.instances["node1"]
minio_client = started_cluster.minio_client
bucket = "warehouse-glue"

test_ref = f"test_optimize_manifest_glue_{uuid.uuid4()}"
table_name = f"{test_ref}_table"
root_namespace = f"{test_ref}_namespace"
table_ref = f"{CATALOG_NAME}.`{root_namespace}.{table_name}`"
write_settings = {"allow_insert_into_iceberg": 1, "write_full_path_in_iceberg_metadata": 1}

create_clickhouse_glue_database(started_cluster, node, CATALOG_NAME)
create_clickhouse_glue_table(started_cluster, node, root_namespace, table_name, "(x String)")

num_inserts = 5
for i in range(num_inserts):
node.query(f"INSERT INTO {table_ref} VALUES ('row{i}');", settings=write_settings)
rows_before = node.query(f"SELECT x FROM {table_ref} ORDER BY x")

def object_key(path):
if path.startswith("s3://"):
path_bucket, key = path[len("s3://"):].split("/", 1)
else:
match = re.match(r"^https?://[^/]+/([^/]+)/(.+)$", path)
assert match, f"unexpected object path {path}"
path_bucket, key = match.group(1), match.group(2)
assert path_bucket == bucket, path
return key

def latest_metadata():
versions = {}
for o in minio_client.list_objects(bucket, prefix=f"{table_name}/metadata/", recursive=True):
match = re.search(r"/v(\d+)[^/]*\.metadata\.json$", o.object_name)
if match:
versions[int(match.group(1))] = o.object_name
assert versions, f"no metadata files for {table_name}"
data = minio_client.get_object(bucket, versions[max(versions)]).read()
if data[:2] == b"\x1f\x8b":
data = gzip.decompress(data)
return json.loads(data)

def read_avro(path):
data = minio_client.get_object(bucket, object_key(path)).read()
with avro.datafile.DataFileReader(io.BytesIO(data), avro.io.DatumReader()) as reader:
return list(reader)

snapshot_id_before = latest_metadata()["current-snapshot-id"]

failpoint = "iceberg_glue_update_table_fail"
node.query(f"SYSTEM ENABLE FAILPOINT {failpoint}")
try:
error = node.query_and_get_error(
f"OPTIMIZE TABLE {table_ref} MANIFEST",
settings={
"allow_experimental_iceberg_compaction": 1,
"iceberg_manifest_min_count_to_compact": 2,
**write_settings,
},
)
finally:
node.query(f"SYSTEM DISABLE FAILPOINT {failpoint}")
assert "rejected the update of the table pointer" in error, error

# The compacted snapshot is the latest one in storage, and every file it names must still exist.
metadata = latest_metadata()
current_snapshot_id = metadata["current-snapshot-id"]
assert current_snapshot_id != snapshot_id_before
current_snapshot = next(s for s in metadata["snapshots"] if s["snapshot-id"] == current_snapshot_id)
assert current_snapshot["summary"]["operation"] == "replace"

manifests = [record["manifest_path"] for record in read_avro(current_snapshot["manifest-list"])]
assert 0 < len(manifests) < num_inserts, f"expected consolidated manifests, got {manifests}"
for manifest in manifests:
minio_client.stat_object(bucket, object_key(manifest))

node.query("SYSTEM DROP ICEBERG METADATA CACHE")

# Readers of the storage layout see the compacted snapshot.
storage_rows = node.query(
f"SELECT x FROM icebergS3('http://minio1:9001/{bucket}/{table_name}/', '{minio_access_key}', '{minio_secret_key}') ORDER BY x"
)
assert storage_rows == rows_before

# The catalog still names the previous metadata, whose files are untouched.
assert node.query(f"SELECT x FROM {table_ref} ORDER BY x") == rows_before
Loading
Loading