diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp index 4afe507ca35c2..ce950432137a8 100644 --- a/src/Common/FailPoint.cpp +++ b/src/Common/FailPoint.cpp @@ -253,6 +253,9 @@ static struct InitFiu PAUSEABLE_ONCE(iceberg_drop_partition_pause_after_discovery) \ ONCE(iceberg_alter_catalog_update_schema_fail) \ ONCE(iceberg_alter_catalog_commit_reported_as_failed) \ + ONCE(iceberg_catalog_update_metadata_fail) \ + ONCE(iceberg_catalog_update_metadata_reported_as_failed) \ + ONCE(iceberg_glue_update_table_fail) \ REGULAR(storage_cluster_read_sleep) \ ONCE(iceberg_writes_non_retry_cleanup) \ ONCE(iceberg_writes_post_publish_throw) \ diff --git a/src/Databases/DataLake/GlueCatalog.cpp b/src/Databases/DataLake/GlueCatalog.cpp index e97d585c1db31..2c9342cc939c5 100644 --- a/src/Databases/DataLake/GlueCatalog.cpp +++ b/src/Databases/DataLake/GlueCatalog.cpp @@ -80,6 +80,7 @@ namespace DB::ErrorCodes namespace DB::FailPoints { extern const char check_database_datalake_negative[]; + extern const char iceberg_glue_update_table_fail[]; } namespace DB::Setting @@ -1234,6 +1235,9 @@ bool GlueCatalog::updateTableInGlue( request.SetTableInput(table_input); + /// Simulates Glue rejecting the update, e.g. because of a `VersionId` mismatch with a concurrent commit. + fiu_do_on(DB::FailPoints::iceberg_glue_update_table_fail, { return false; }); + Aws::Glue::Model::UpdateTableOutcome response; { diff --git a/src/Databases/DataLake/RestCatalog.cpp b/src/Databases/DataLake/RestCatalog.cpp index 6160a8bb57ed2..0f73d429bc07a 100644 --- a/src/Databases/DataLake/RestCatalog.cpp +++ b/src/Databases/DataLake/RestCatalog.cpp @@ -94,6 +94,8 @@ namespace DB::FailPoints extern const char check_database_datalake_negative[]; extern const char iceberg_alter_catalog_update_schema_fail[]; extern const char iceberg_alter_catalog_commit_reported_as_failed[]; + extern const char iceberg_catalog_update_metadata_fail[]; + extern const char iceberg_catalog_update_metadata_reported_as_failed[]; } namespace ProfileEvents @@ -2766,6 +2768,9 @@ bool RestCatalog::updateMetadata(const String & namespace_name, const String & t auto request_body = buildUpdateMetadataRequestBody(namespace_name, table_name, new_snapshot); + /// Simulates a retryable failure where the catalog did not apply the update. + fiu_do_on(DB::FailPoints::iceberg_catalog_update_metadata_fail, { return false; }); + try { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUpdateTable); @@ -2791,6 +2796,11 @@ bool RestCatalog::updateMetadata(const String & namespace_name, const String & t table_name, ex.displayText()); } + + /// Simulates the Iceberg "commit state unknown" case: the catalog applied the update but the + /// client observes a failure, e.g. because a proxy turned the response into a 5xx. + fiu_do_on(DB::FailPoints::iceberg_catalog_update_metadata_reported_as_failed, { return false; }); + return true; } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp index 941b9d1a69f5f..22bfd0ac2f97f 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -8,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -42,6 +44,7 @@ namespace DB::ErrorCodes { extern const int BAD_ARGUMENTS; + extern const int DATALAKE_DATABASE_ERROR; extern const int LOGICAL_ERROR; extern const int CANNOT_UNLINK; extern const int ICEBERG_SPECIFICATION_VIOLATION; @@ -488,6 +491,53 @@ static void writeDataFiles( } } +/// Whether the table metadata that `catalog` currently points to contains a snapshot with `snapshot_id`. +/// Used to resolve the "commit state unknown" case, where the catalog may have applied an update +/// although the client observed a failure (e.g. a proxy turning the response into a 5xx). +/// Throws if the catalog or the metadata file cannot be read, so that the caller never treats +/// an unknown state as "not committed". +static bool isSnapshotInCatalogMetadata( + const std::shared_ptr & catalog, + const StorageID & table_id, + Int64 snapshot_id, + const PersistentTableComponents & persistent_table_components, + const DataLakeStorageSettings & data_lake_settings, + ObjectStoragePtr object_storage, + ContextPtr context, + LoggerPtr log) +{ + const auto metadata_file_info = getLatestMetadataFileAndVersionWithCatalog( + object_storage, + catalog, + table_id.getTableName(), + persistent_table_components.table_path, + data_lake_settings, + persistent_table_components.metadata_cache, + context, + log.get(), + persistent_table_components.table_uuid, + persistent_table_components.metadata_compression_method); + auto metadata_object = getMetadataJSONObject( + metadata_file_info.path, + object_storage, + persistent_table_components.metadata_cache, + context, + log, + metadata_file_info.compression_method, + persistent_table_components.table_uuid); + + if (!metadata_object->has(Iceberg::f_snapshots)) + return false; + + auto snapshots = metadata_object->get(Iceberg::f_snapshots).extract(); + for (size_t i = 0; i < snapshots->size(); ++i) + { + if (snapshots->getObject(static_cast(i))->getValue(Iceberg::f_metadata_snapshot_id) == snapshot_id) + return true; + } + return false; +} + static bool writeConsolidatedManifestFile( int metadata_version, const String & metadata_file_path, @@ -832,9 +882,21 @@ static bool writeConsolidatedManifestFile( /// Parallel to consolidated_manifest_paths: each manifest's partition fields (value + type), used to recompute the manifest-list `partitions` summary. std::vector>> entry_partition_summaries; + /// Whether the new snapshot may be visible to readers: through the metadata file and version hint + /// in storage, or through the catalog pointer. From then on the new manifests and manifest list may be + /// referenced by the current snapshot, so they must never be deleted: an orphaned file is harmless, + /// a deleted live one makes the table unreadable. + bool published = false; + /// Cleanup for both commit conflict and exceptions; paths are tracked before writeObject so partially-created objects are removed (removeObjectIfExists tolerates missing objects). auto cleanup = [&]() { + if (published) + { + LOG_INFO(log, "Not removing the new manifest files: the new snapshot may already be visible"); + return; + } + for (const auto & mp : consolidated_manifest_paths) { try @@ -1009,16 +1071,68 @@ static bool writeConsolidatedManifestFile( return false; } + /// The metadata file and version hint in storage now name the new snapshot. + if (!catalog_writes_metadata_file) + published = true; + /// Advance the catalog pointer to the new metadata so catalog-based readers see the compacted snapshot. if (catalog) { auto catalog_filename = path_resolver.resolveForCatalog(generated_metadata_info.path); const auto & [namespace_name, table_name] = DataLake::parseTableName(table_id.getTableName()); - if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot.snapshot, context->getForwardedAuthToken())) + + if (!catalog_writes_metadata_file) { - LOG_INFO(log, "Metadata commit conflict detected via catalog, cleaning up temporary files"); - cleanup(); - return false; + /// The snapshot is already visible through storage, so a failed catalog update cannot be + /// undone by deleting files. Retrying is not correct either: the retry would read the + /// compacted metadata, find nothing to do and report success while the catalog pointer + /// still names the previous metadata. Keep the files and report the error. + if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot.snapshot, context->getForwardedAuthToken())) + throw Exception( + ErrorCodes::DATALAKE_DATABASE_ERROR, + "Compacted Iceberg metadata {} of table {}.{} was committed to storage, but the catalog " + "rejected the update of the table pointer. The table in storage is consistent; " + "the catalog still names the previous metadata", + catalog_filename, namespace_name, table_name); + } + else + { + bool updated = false; + std::exception_ptr update_exception; + try + { + updated = catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot.snapshot, context->getForwardedAuthToken()); + } + catch (...) + { + update_exception = std::current_exception(); + } + + if (!updated) + { + /// Commit state unknown: the catalog may have applied the update although the client + /// saw a failure. Until the catalog proves otherwise, the new files may be live, so + /// keep them even if the check below throws. + published = true; + const Int64 new_snapshot_id = new_snapshot.snapshot->getValue(Iceberg::f_metadata_snapshot_id); + if (isSnapshotInCatalogMetadata( + catalog, table_id, new_snapshot_id, persistent_table_components, data_lake_settings, object_storage, context, log)) + { + LOG_INFO(log, "Catalog update for snapshot {} was reported as failed, but the catalog has applied it", new_snapshot_id); + } + else + { + /// The update did not land: the new files are not referenced by any snapshot. + published = false; + if (update_exception) + std::rethrow_exception(update_exception); + + LOG_INFO(log, "Metadata commit conflict detected via catalog, cleaning up temporary files"); + cleanup(); + return false; + } + } + published = true; } } } @@ -1536,17 +1650,19 @@ void compactIcebergManifests( if (attempt > 0) LOG_INFO(log, "Retrying manifest compaction (attempt {}/{})", attempt + 1, MAX_COMPACTION_RETRIES); - const auto [metadata_version, metadata_file_path, _] = getLatestOrExplicitMetadataFileAndVersion( + /// With a catalog, the catalog names the current metadata file. Listing the storage instead may pick the + /// metadata of another table that shares the location, or miss a commit that is visible only in the catalog. + const auto [metadata_version, metadata_file_path, _] = getLatestMetadataFileAndVersionWithCatalog( object_storage_, + catalog, + table_id.getTableName(), persistent_table_components.table_path, data_lake_settings, persistent_table_components.metadata_cache, context_, log.get(), persistent_table_components.table_uuid, - persistent_table_components.metadata_compression_method, - /* force_fetch_latest_metadata */ true, - /* ignore_explicit_metadata_file_path */ true); + persistent_table_components.metadata_compression_method); auto metadata_object = getMetadataJSONObject( metadata_file_path, diff --git a/tests/integration/test_database_glue/test.py b/tests/integration/test_database_glue/test.py index f5d4d2027619d..72825b202d7fc 100644 --- a/tests/integration/test_database_glue/test.py +++ b/tests/integration/test_database_glue/test.py @@ -2251,3 +2251,104 @@ def test_catalog_schema_with_empty_column_name_is_rejected(started_cluster): assert node.query("SELECT 1") == "1\n" node.query(f"DROP DATABASE IF EXISTS {db_name} SYNC") + + +def test_optimize_manifest_glue_update_fails(started_cluster): + """`OPTIMIZE TABLE ... MANIFEST` must not delete the files of a snapshot that is visible in storage. + + For Glue the compacted metadata file is written to storage before the catalog pointer is updated. + When Glue then rejects the update (e.g. a `VersionId` mismatch with a concurrent commit), the new + snapshot is already the latest one in storage, so its manifests must stay in place. + https://github.com/Altinity/ClickHouse/issues/2416 + """ + import gzip + import io + import re + + import avro.datafile + import avro.io + + node = started_cluster.instances["node1"] + minio_client = started_cluster.minio_client + bucket = "warehouse-glue" + + test_ref = f"test_optimize_manifest_glue_{uuid.uuid4()}" + table_name = f"{test_ref}_table" + root_namespace = f"{test_ref}_namespace" + table_ref = f"{CATALOG_NAME}.`{root_namespace}.{table_name}`" + write_settings = {"allow_insert_into_iceberg": 1, "write_full_path_in_iceberg_metadata": 1} + + create_clickhouse_glue_database(started_cluster, node, CATALOG_NAME) + create_clickhouse_glue_table(started_cluster, node, root_namespace, table_name, "(x String)") + + num_inserts = 5 + for i in range(num_inserts): + node.query(f"INSERT INTO {table_ref} VALUES ('row{i}');", settings=write_settings) + rows_before = node.query(f"SELECT x FROM {table_ref} ORDER BY x") + + def object_key(path): + if path.startswith("s3://"): + path_bucket, key = path[len("s3://"):].split("/", 1) + else: + match = re.match(r"^https?://[^/]+/([^/]+)/(.+)$", path) + assert match, f"unexpected object path {path}" + path_bucket, key = match.group(1), match.group(2) + assert path_bucket == bucket, path + return key + + def latest_metadata(): + versions = {} + for o in minio_client.list_objects(bucket, prefix=f"{table_name}/metadata/", recursive=True): + match = re.search(r"/v(\d+)[^/]*\.metadata\.json$", o.object_name) + if match: + versions[int(match.group(1))] = o.object_name + assert versions, f"no metadata files for {table_name}" + data = minio_client.get_object(bucket, versions[max(versions)]).read() + if data[:2] == b"\x1f\x8b": + data = gzip.decompress(data) + return json.loads(data) + + def read_avro(path): + data = minio_client.get_object(bucket, object_key(path)).read() + with avro.datafile.DataFileReader(io.BytesIO(data), avro.io.DatumReader()) as reader: + return list(reader) + + snapshot_id_before = latest_metadata()["current-snapshot-id"] + + failpoint = "iceberg_glue_update_table_fail" + node.query(f"SYSTEM ENABLE FAILPOINT {failpoint}") + try: + error = node.query_and_get_error( + f"OPTIMIZE TABLE {table_ref} MANIFEST", + settings={ + "allow_experimental_iceberg_compaction": 1, + "iceberg_manifest_min_count_to_compact": 2, + **write_settings, + }, + ) + finally: + node.query(f"SYSTEM DISABLE FAILPOINT {failpoint}") + assert "rejected the update of the table pointer" in error, error + + # The compacted snapshot is the latest one in storage, and every file it names must still exist. + metadata = latest_metadata() + current_snapshot_id = metadata["current-snapshot-id"] + assert current_snapshot_id != snapshot_id_before + current_snapshot = next(s for s in metadata["snapshots"] if s["snapshot-id"] == current_snapshot_id) + assert current_snapshot["summary"]["operation"] == "replace" + + manifests = [record["manifest_path"] for record in read_avro(current_snapshot["manifest-list"])] + assert 0 < len(manifests) < num_inserts, f"expected consolidated manifests, got {manifests}" + for manifest in manifests: + minio_client.stat_object(bucket, object_key(manifest)) + + node.query("SYSTEM DROP ICEBERG METADATA CACHE") + + # Readers of the storage layout see the compacted snapshot. + storage_rows = node.query( + f"SELECT x FROM icebergS3('http://minio1:9001/{bucket}/{table_name}/', '{minio_access_key}', '{minio_secret_key}') ORDER BY x" + ) + assert storage_rows == rows_before + + # The catalog still names the previous metadata, whose files are untouched. + assert node.query(f"SELECT x FROM {table_ref} ORDER BY x") == rows_before diff --git a/tests/integration/test_database_iceberg/test.py b/tests/integration/test_database_iceberg/test.py index 98d41bad325a2..2b0d4a8693547 100644 --- a/tests/integration/test_database_iceberg/test.py +++ b/tests/integration/test_database_iceberg/test.py @@ -992,6 +992,209 @@ def current_snapshot_id(): assert rows_after == rows_before +def _split_object_path(path): + """Split `s3://bucket/key` or `http://host:port/bucket/key` into (bucket, key).""" + if path.startswith("s3://"): + return path[len("s3://"):].split("/", 1) + match = re.match(r"^https?://[^/]+/([^/]+)/(.+)$", path) + assert match, f"unexpected object path {path}" + return match.group(1), match.group(2) + + +def _read_avro_records(minio_client, path): + # pyiceberg rejects the manifest lists written by ClickHouse (no field-ids), so read them directly. + import avro.datafile + import avro.io + + bucket, key = _split_object_path(path) + data = minio_client.get_object(bucket, key).read() + with avro.datafile.DataFileReader(io.BytesIO(data), avro.io.DatumReader()) as reader: + return list(reader) + + +def _create_table_for_manifest_compaction( + started_cluster, node, test_ref, num_inserts=5, table_suffix="table", location=None +): + table_name = f"{test_ref}_{table_suffix}" + root_namespace = f"{test_ref}_namespace" + + catalog = load_catalog_impl(started_cluster) + if (root_namespace,) not in catalog.list_namespaces(): + catalog.create_namespace(root_namespace) + # Unpartitioned table, so every per-insert data manifest can consolidate into a single one. + # Own location by default: `create_table` puts every table at the same location, so files of other tests + # would be mixed with the files of this table. + catalog.create_table( + identifier=f"{root_namespace}.{table_name}", + schema=DEFAULT_SCHEMA, + location=location or f"s3://warehouse-rest/{table_name}", + partition_spec=PartitionSpec(), + sort_order=DEFAULT_SORT_ORDER, + ) + create_clickhouse_iceberg_database(started_cluster, node, CATALOG_NAME) + + table_ref = f"{CATALOG_NAME}.`{root_namespace}.{table_name}`" + for i in range(num_inserts): + node.query( + f"INSERT INTO {table_ref} VALUES (NULL, 'sym{i}', {100 + i}, {200 + i}, tuple('bot'));", + settings={"allow_insert_into_iceberg": 1, "write_full_path_in_iceberg_metadata": 1}, + ) + return catalog, f"{root_namespace}.{table_name}", table_ref + + +def _optimize_manifest_with_failpoint(node, table_ref, failpoint): + node.query(f"SYSTEM ENABLE FAILPOINT {failpoint}") + try: + node.query( + f"OPTIMIZE TABLE {table_ref} MANIFEST", + settings={ + "allow_experimental_iceberg_compaction": 1, + "iceberg_manifest_min_count_to_compact": 2, + "allow_insert_into_iceberg": 1, + "write_full_path_in_iceberg_metadata": 1, + }, + ) + finally: + node.query(f"SYSTEM DISABLE FAILPOINT {failpoint}") + + +def _check_current_snapshot_files_exist(started_cluster, catalog, table_identifier, num_inserts): + """Every manifest and the manifest list named by the current snapshot must still exist.""" + minio_client = started_cluster.minio_client + metadata = catalog.load_table(table_identifier).metadata + manifest_list = metadata.snapshot_by_id(metadata.current_snapshot_id).manifest_list + manifests = [record["manifest_path"] for record in _read_avro_records(minio_client, manifest_list)] + assert 0 < len(manifests) < num_inserts, f"expected consolidated manifests, got {manifests}" + for manifest in manifests: + minio_client.stat_object(*_split_object_path(manifest)) + + +def test_optimize_manifest_commit_state_unknown(started_cluster): + """`OPTIMIZE TABLE ... MANIFEST` must not delete the files of a snapshot the catalog has applied. + + The failpoint makes the REST catalog apply the update and then report a retryable failure + (commit state unknown, e.g. a proxy turning the response into a 5xx). The compaction must + notice that its snapshot is current instead of removing the manifests it names. + https://github.com/Altinity/ClickHouse/issues/2416 + """ + node = started_cluster.instances["node1"] + num_inserts = 5 + catalog, table_identifier, table_ref = _create_table_for_manifest_compaction( + started_cluster, node, f"test_optimize_manifest_unknown_{uuid.uuid4()}", num_inserts + ) + + metadata_before = catalog.load_table(table_identifier).metadata + rows_before = node.query(f"SELECT symbol, bid, ask FROM {table_ref} ORDER BY ALL") + + _optimize_manifest_with_failpoint(node, table_ref, "iceberg_catalog_update_metadata_reported_as_failed") + + metadata_after = catalog.load_table(table_identifier).metadata + assert metadata_after.current_snapshot_id != metadata_before.current_snapshot_id + # The applied commit is detected, so there is no second `replace` snapshot from a retry. + assert len(metadata_after.snapshots) == len(metadata_before.snapshots) + 1 + + _check_current_snapshot_files_exist(started_cluster, catalog, table_identifier, num_inserts) + + node.query("SYSTEM DROP ICEBERG METADATA CACHE") + assert node.query(f"SELECT symbol, bid, ask FROM {table_ref} ORDER BY ALL") == rows_before + + +def test_optimize_manifest_catalog_conflict_cleans_up(started_cluster): + """When the catalog update really does not apply, the compaction removes its files and retries. + + https://github.com/Altinity/ClickHouse/issues/2416 + """ + node = started_cluster.instances["node1"] + num_inserts = 5 + catalog, table_identifier, table_ref = _create_table_for_manifest_compaction( + started_cluster, node, f"test_optimize_manifest_conflict_{uuid.uuid4()}", num_inserts + ) + + metadata_before = catalog.load_table(table_identifier).metadata + rows_before = node.query(f"SELECT symbol, bid, ask FROM {table_ref} ORDER BY ALL") + + _optimize_manifest_with_failpoint(node, table_ref, "iceberg_catalog_update_metadata_fail") + + table = catalog.load_table(table_identifier) + metadata_after = table.metadata + assert metadata_after.current_snapshot_id != metadata_before.current_snapshot_id + assert len(metadata_after.snapshots) == len(metadata_before.snapshots) + 1 + + _check_current_snapshot_files_exist(started_cluster, catalog, table_identifier, num_inserts) + + # The rejected attempt's manifests and manifest list are not referenced by any snapshot, so they must be gone. + minio_client = started_cluster.minio_client + referenced = set() + for snapshot in metadata_after.snapshots: + referenced.add(_split_object_path(snapshot.manifest_list)[1]) + for record in _read_avro_records(minio_client, snapshot.manifest_list): + referenced.add(_split_object_path(record["manifest_path"])[1]) + + bucket, table_prefix = _split_object_path(table.location()) + metadata_prefix = table_prefix.rstrip("/") + "/metadata/" + avro_objects = { + o.object_name + for o in minio_client.list_objects(bucket, prefix=metadata_prefix, recursive=True) + if o.object_name.endswith(".avro") + } + orphans = avro_objects - referenced + assert not orphans, f"rejected compaction attempt left files behind: {sorted(orphans)}" + + node.query("SYSTEM DROP ICEBERG METADATA CACHE") + assert node.query(f"SELECT symbol, bid, ask FROM {table_ref} ORDER BY ALL") == rows_before + + +def test_optimize_manifest_uses_catalog_metadata(started_cluster): + """`OPTIMIZE TABLE ... MANIFEST` must compact the metadata the catalog names for the table. + + Two tables share one location. The other table has more metadata versions, so the latest metadata file + found by listing the storage belongs to it. Compacting the first table must not pick that file. + https://github.com/Altinity/ClickHouse/issues/2416 + """ + node = started_cluster.instances["node1"] + test_ref = f"test_optimize_manifest_shared_location_{uuid.uuid4()}" + location = f"s3://warehouse-rest/{test_ref}" + num_inserts = 5 + catalog, table_identifier, table_ref = _create_table_for_manifest_compaction( + started_cluster, node, test_ref, num_inserts, table_suffix="table", location=location + ) + _, other_identifier, other_ref = _create_table_for_manifest_compaction( + started_cluster, node, test_ref, num_inserts + 3, table_suffix="other", location=location + ) + + metadata_before = catalog.load_table(table_identifier).metadata + other_metadata_before = catalog.load_table(other_identifier).metadata + rows_before = node.query(f"SELECT symbol, bid, ask FROM {table_ref} ORDER BY ALL") + other_rows_before = node.query(f"SELECT symbol, bid, ask FROM {other_ref} ORDER BY ALL") + + node.query( + f"OPTIMIZE TABLE {table_ref} MANIFEST", + settings={ + "allow_experimental_iceberg_compaction": 1, + "iceberg_manifest_min_count_to_compact": 2, + "allow_insert_into_iceberg": 1, + "write_full_path_in_iceberg_metadata": 1, + }, + ) + + metadata_after = catalog.load_table(table_identifier).metadata + assert metadata_after.current_snapshot_id != metadata_before.current_snapshot_id + assert len(metadata_after.snapshots) == len(metadata_before.snapshots) + 1 + # The new snapshot must descend from this table's own history, not from the other table's. + assert ( + metadata_after.snapshot_by_id(metadata_after.current_snapshot_id).parent_snapshot_id + == metadata_before.current_snapshot_id + ) + _check_current_snapshot_files_exist(started_cluster, catalog, table_identifier, num_inserts) + + other_metadata_after = catalog.load_table(other_identifier).metadata + assert other_metadata_after.current_snapshot_id == other_metadata_before.current_snapshot_id + + node.query("SYSTEM DROP ICEBERG METADATA CACHE") + assert node.query(f"SELECT symbol, bid, ask FROM {table_ref} ORDER BY ALL") == rows_before + assert node.query(f"SELECT symbol, bid, ask FROM {other_ref} ORDER BY ALL") == other_rows_before + + @pytest.mark.parametrize( "fields_to_remove", [