diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 7b4cfb0b38b8d..263040f731b11 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -9290,6 +9290,9 @@ Default age threshold in seconds for orphan file removal in Iceberg tables. File )", EXPERIMENTAL) \ DECLARE(Bool, allow_experimental_expire_snapshots, false, R"( Allow to execute experimental Iceberg command `ALTER TABLE ... EXECUTE expire_snapshots`. +)", EXPERIMENTAL) \ + DECLARE(Bool, allow_experimental_iceberg_add_files, false, R"( +Allow to execute experimental Iceberg command `ALTER TABLE ... EXECUTE add_files`, which registers existing Parquet files under the table location without rewriting them. )", EXPERIMENTAL) \ DECLARE(Bool, write_full_path_in_iceberg_metadata, false, R"( Write full paths (including s3://) into iceberg metadata files. diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 39a666ae61346..432241795742a 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -54,6 +54,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() { {"input_format_parquet_detect_variant_by_structure", false, true, "New setting to read Spark variant columns, which have no `VARIANT` logical type, as `Dynamic` instead of `Tuple`"}, {"use_iceberg_manifest_list_partition_pruning", false, true, "New setting to skip Iceberg manifest files whose manifest-list partition summaries cannot match the query filter, without reading them."}, + {"allow_experimental_iceberg_add_files", false, false, "New setting to gate the experimental Iceberg command `ALTER TABLE ... EXECUTE add_files`."}, }); addSettingsChanges(settings_changes_history, "26.8.27.20001.altinityantalya", { diff --git a/src/Databases/DataLake/ICatalog.cpp b/src/Databases/DataLake/ICatalog.cpp index 9cc28afc48917..afa6feceba484 100644 --- a/src/Databases/DataLake/ICatalog.cpp +++ b/src/Databases/DataLake/ICatalog.cpp @@ -523,6 +523,20 @@ bool ICatalog::updateMetadata(const String & /*namespace_name*/, const String & throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "updateMetadata is not implemented"); } +bool ICatalog::updateMetadataAndProperties( + const String & namespace_name, + const String & table_name, + const String & new_metadata_path, + Poco::JSON::Object::Ptr new_snapshot, + const std::map & properties_to_set, + const DB::ForwardedAuthTokenPtr & auth_token) const +{ + if (!properties_to_set.empty() && isTransactional()) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "Setting table properties together with a snapshot is not implemented for this catalog"); + return updateMetadata(namespace_name, table_name, new_metadata_path, new_snapshot, auth_token); +} + bool ICatalog::updateSchema( const String & /*namespace_name*/, const String & /*table_name*/, diff --git a/src/Databases/DataLake/ICatalog.h b/src/Databases/DataLake/ICatalog.h index 5e092e92b75f2..0fd70ad24de41 100644 --- a/src/Databases/DataLake/ICatalog.h +++ b/src/Databases/DataLake/ICatalog.h @@ -15,6 +15,7 @@ #include #include +#include #include #include @@ -333,6 +334,18 @@ class ICatalog /// Updates metadata in catalog. virtual bool updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot, const DB::ForwardedAuthTokenPtr & auth_token) const; + /// Like `updateMetadata`, and also sets the table properties `properties_to_set` in the same commit. + /// A non-transactional catalog only stores a pointer to `new_metadata_path`, which already holds the + /// properties, so the default implementation delegates to `updateMetadata` for it. A transactional + /// catalog applies the updates itself and must override this to send the properties. + virtual bool updateMetadataAndProperties( + const String & namespace_name, + const String & table_name, + const String & new_metadata_path, + Poco::JSON::Object::Ptr new_snapshot, + const std::map & properties_to_set, + const DB::ForwardedAuthTokenPtr & auth_token) const; + /// Commit a schema evolution (ADD/DROP/MODIFY/RENAME COLUMN) to the catalog. /// `new_metadata_path` is the path of the freshly written `vN.metadata.json`; it is used by /// non-transactional catalogs (e.g. Glue) that only store a pointer to the metadata file. diff --git a/src/Databases/DataLake/RestCatalog.cpp b/src/Databases/DataLake/RestCatalog.cpp index 6160a8bb57ed2..8c7a6d0d1f529 100644 --- a/src/Databases/DataLake/RestCatalog.cpp +++ b/src/Databases/DataLake/RestCatalog.cpp @@ -429,7 +429,10 @@ Poco::JSON::Object::Ptr buildUpdateSchemaRequestBody( } Poco::JSON::Object::Ptr buildUpdateMetadataRequestBody( - const String & namespace_name, const String & table_name, Poco::JSON::Object::Ptr new_snapshot) + const String & namespace_name, + const String & table_name, + Poco::JSON::Object::Ptr new_snapshot, + const std::map & properties_to_set) { if (!new_snapshot) return nullptr; @@ -475,6 +478,16 @@ Poco::JSON::Object::Ptr buildUpdateMetadataRequestBody( set_snapshot->set("snapshot-id", new_snapshot->getValue("snapshot-id")); updates->add(set_snapshot); } + if (!properties_to_set.empty()) + { + Poco::JSON::Object::Ptr set_properties = new Poco::JSON::Object; + set_properties->set("action", "set-properties"); + Poco::JSON::Object::Ptr properties = new Poco::JSON::Object; + for (const auto & [key, value] : properties_to_set) + properties->set(key, value); + set_properties->set("updates", properties); + updates->add(set_properties); + } request_body->set("updates", updates); return request_body; @@ -2751,7 +2764,18 @@ bool RestCatalog::createTable( } -bool RestCatalog::updateMetadata(const String & namespace_name, const String & table_name, const String & /*new_metadata_path*/, Poco::JSON::Object::Ptr new_snapshot, const DB::ForwardedAuthTokenPtr & auth_token) const +bool RestCatalog::updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot, const DB::ForwardedAuthTokenPtr & auth_token) const +{ + return updateMetadataAndProperties(namespace_name, table_name, new_metadata_path, new_snapshot, /* properties_to_set */ {}, auth_token); +} + +bool RestCatalog::updateMetadataAndProperties( + const String & namespace_name, + const String & table_name, + const String & /*new_metadata_path*/, + Poco::JSON::Object::Ptr new_snapshot, + const std::map & properties_to_set, + const DB::ForwardedAuthTokenPtr & auth_token) const { loadConfigIfNeeded(auth_token); @@ -2764,7 +2788,7 @@ bool RestCatalog::updateMetadata(const String & namespace_name, const String & t const auto state_snapshot = getStateSnapshot(); const std::string endpoint = (base_url / state_snapshot->config.prefix / NAMESPACES_ENDPOINT / encodeNamespaceForURI(namespace_name) / "tables" / table_name).generic_string(); - auto request_body = buildUpdateMetadataRequestBody(namespace_name, table_name, new_snapshot); + auto request_body = buildUpdateMetadataRequestBody(namespace_name, table_name, new_snapshot, properties_to_set); try { diff --git a/src/Databases/DataLake/RestCatalog.h b/src/Databases/DataLake/RestCatalog.h index f088e0c6f658f..1d3bccf5ddf17 100644 --- a/src/Databases/DataLake/RestCatalog.h +++ b/src/Databases/DataLake/RestCatalog.h @@ -121,6 +121,14 @@ class RestCatalog : public ICatalog, public DB::WithContext bool updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot, const DB::ForwardedAuthTokenPtr & auth_token) const override; + bool updateMetadataAndProperties( + const String & namespace_name, + const String & table_name, + const String & new_metadata_path, + Poco::JSON::Object::Ptr new_snapshot, + const std::map & properties_to_set, + const DB::ForwardedAuthTokenPtr & auth_token) const override; + bool updateSchema( const String & namespace_name, const String & table_name, @@ -609,10 +617,13 @@ Poco::JSON::Object::Ptr buildUpdateSchemaRequestBody( Int32 previous_schema_id, Int32 new_last_column_id); +/// Builds the JSON body that commits `new_snapshot` (and sets `properties_to_set` in the same commit) +/// via the Iceberg REST catalog. Poco::JSON::Object::Ptr buildUpdateMetadataRequestBody( const String & namespace_name, const String & table_name, - Poco::JSON::Object::Ptr new_snapshot); + Poco::JSON::Object::Ptr new_snapshot, + const std::map & properties_to_set = {}); } diff --git a/src/Databases/DataLake/UnityV2Catalog.cpp b/src/Databases/DataLake/UnityV2Catalog.cpp index 14a497d1d3735..33facbb7ef837 100644 --- a/src/Databases/DataLake/UnityV2Catalog.cpp +++ b/src/Databases/DataLake/UnityV2Catalog.cpp @@ -439,6 +439,21 @@ bool UnityV2Catalog::updateMetadata( }); } +bool UnityV2Catalog::updateMetadataAndProperties( + const String & namespace_name, + const String & table_name, + const String & new_metadata_path, + Poco::JSON::Object::Ptr new_snapshot, + const std::map & properties_to_set, + const DB::ForwardedAuthTokenPtr & auth_token) const +{ + return requestWithRetry([&](bool force_refresh) + { + return getIcebergRestCatalog(force_refresh)->updateMetadataAndProperties( + namespace_name, table_name, new_metadata_path, new_snapshot, properties_to_set, auth_token); + }); +} + bool UnityV2Catalog::updateSchema( const String & namespace_name, const String & table_name, diff --git a/src/Databases/DataLake/UnityV2Catalog.h b/src/Databases/DataLake/UnityV2Catalog.h index d40a90f125c22..672741699a65c 100644 --- a/src/Databases/DataLake/UnityV2Catalog.h +++ b/src/Databases/DataLake/UnityV2Catalog.h @@ -91,6 +91,14 @@ class UnityV2Catalog final : public ICatalog, private DB::WithContext Poco::JSON::Object::Ptr new_snapshot, const DB::ForwardedAuthTokenPtr & auth_token) const override; + bool updateMetadataAndProperties( + const String & namespace_name, + const String & table_name, + const String & new_metadata_path, + Poco::JSON::Object::Ptr new_snapshot, + const std::map & properties_to_set, + const DB::ForwardedAuthTokenPtr & auth_token) const override; + bool updateSchema( const String & namespace_name, const String & table_name, diff --git a/src/Databases/DataLake/tests/gtest_rest_catalog_update_metadata.cpp b/src/Databases/DataLake/tests/gtest_rest_catalog_update_metadata.cpp index 8b209262cf628..50541508dd6c5 100644 --- a/src/Databases/DataLake/tests/gtest_rest_catalog_update_metadata.cpp +++ b/src/Databases/DataLake/tests/gtest_rest_catalog_update_metadata.cpp @@ -131,6 +131,28 @@ TEST(RestCatalogUpdateMetadataBody, SnapshotUpdateParentMinusOneNoRequirement) EXPECT_FALSE(req->has("snapshot-id")); } +TEST(RestCatalogUpdateMetadataBody, PropertiesAreSetInTheSameCommit) +{ + Poco::JSON::Object::Ptr snapshot = new Poco::JSON::Object; + snapshot->set("snapshot-id", static_cast(7)); + + auto without_properties = DataLake::buildUpdateMetadataRequestBody("ns", "t", snapshot); + ASSERT_TRUE(without_properties); + EXPECT_FALSE(findUpdateByAction(without_properties->getArray("updates"), "set-properties")); + + auto body = DataLake::buildUpdateMetadataRequestBody( + "ns", "t", snapshot, {{"schema.name-mapping.default", R"([{"field-id":1,"names":["a"]}])"}}); + ASSERT_TRUE(body); + auto updates = body->getArray("updates"); + ASSERT_TRUE(findUpdateByAction(updates, "add-snapshot")); + ASSERT_TRUE(findUpdateByAction(updates, "set-snapshot-ref")); + auto set_properties = findUpdateByAction(updates, "set-properties"); + ASSERT_TRUE(set_properties); + EXPECT_EQ( + set_properties->getObject("updates")->getValue("schema.name-mapping.default"), + R"([{"field-id":1,"names":["a"]}])"); +} + TEST(RestCatalogUpdateSchemaBody, EquivalentSchemaDeduplicates) { Poco::JSON::Object::Ptr metadata = new Poco::JSON::Object; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/AddFilesExecute.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/AddFilesExecute.cpp new file mode 100644 index 0000000000000..a56f7e716a494 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/AddFilesExecute.cpp @@ -0,0 +1,1480 @@ +#include + +#include "config.h" + +#if USE_AVRO + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ +extern const int BAD_ARGUMENTS; +extern const int DATALAKE_DATABASE_ERROR; +extern const int ICEBERG_SPECIFICATION_VIOLATION; +extern const int LIMIT_EXCEEDED; +extern const int LOGICAL_ERROR; +extern const int METADATA_MISMATCH; +extern const int SUPPORT_IS_DISABLED; +} + +namespace DataLakeStorageSetting +{ +extern const DataLakeStorageSettingsBool iceberg_use_version_hint; +} + +namespace Iceberg +{ + +namespace +{ + +ExecuteCommandArgs makeSchema() +{ + ExecuteCommandArgs schema("add_files"); + schema.addPositional("paths"); + schema.addNamed("check_duplicate_files", Field::Types::UInt64); + schema.addNamed("allow_missing_columns", Field::Types::UInt64); + schema.addDefault("check_duplicate_files", Field(UInt64(1))); + schema.addDefault("allow_missing_columns", Field(UInt64(0))); + return schema; +} + +std::vector parsePathsArgument(const Field & field) +{ + std::vector paths; + if (field.getType() == Field::Types::String) + { + paths.push_back(field.safeGet()); + } + else if (field.getType() == Field::Types::Array) + { + for (const auto & item : field.safeGet()) + paths.push_back(item.safeGet()); + } + else + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "add_files expects 'paths' to be a String or an Array of Strings, got {}", field.getTypeName()); + } + if (paths.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "add_files expects at least one path"); + return paths; +} + +bool hasHiddenPathComponent(const String & path) +{ + size_t start = 0; + while (start <= path.size()) + { + size_t slash = path.find('/', start); + String component = path.substr(start, slash == String::npos ? String::npos : slash - start); + if (!component.empty() && component[0] == '.') + return true; + if (slash == String::npos) + break; + start = slash + 1; + } + return false; +} + +bool endsWithParquet(const String & path) +{ + return Poco::toLower(path).ends_with(".parquet"); +} + +/// The files are read and registered through the table's storage and credentials, so only files under +/// the table location may be registered. Otherwise `IcebergPathResolver::resolve` maps an outside path +/// to some key of the table's storage, which lets a user expose objects the table does not own, and the +/// manifest records a path that names a different object than the one ClickHouse read. +void checkPathUnderTableLocation(const String & path, const String & table_location) +{ + if (table_location.empty() || !path.starts_with(table_location) || path.size() <= table_location.size() + || path[table_location.size()] != '/') + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "add_files: path '{}' is not under the table location '{}'; only files under the table location can be registered", + path, table_location); + + std::string_view rest = std::string_view(path).substr(table_location.size() + 1); + while (true) + { + size_t slash = rest.find('/'); + std::string_view component = rest.substr(0, slash); + if (component == "." || component == "..") + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "add_files: path '{}' contains a '{}' component; give the path without relative components", path, component); + if (slash == std::string_view::npos) + break; + rest = rest.substr(slash + 1); + } +} + +/// The Iceberg `truncate[N]` transform of a string keeps the first N code points, as `icebergTruncate` does. +String truncateToCodePoints(const String & value, size_t code_points) +{ + size_t pos = 0; + for (size_t i = 0; i < code_points && pos < value.size(); ++i) + pos += UTF8::seqLength(static_cast(value[pos])); + return value.substr(0, std::min(pos, value.size())); +} + +/// Leaf fields (over nested structs) that must have a value in every row: the field and all its +/// ancestors are required. A file without such a column cannot be registered. +std::unordered_set collectRequiredFieldIds(Poco::JSON::Array::Ptr schema_fields) +{ + std::unordered_set result; + std::function collect; + collect = [&](Poco::JSON::Object::Ptr field) + { + if (!field->has(Iceberg::f_required) || !field->getValue(Iceberg::f_required)) + return; + auto type_var = field->get(Iceberg::f_type); + if (type_var.isString()) + { + result.insert(field->getValue(Iceberg::f_id)); + return; + } + const auto & type_obj = type_var.extract(); + if (type_obj->getValue(Iceberg::f_type) == "struct") + { + for (const auto & nested : *type_obj->getArray(Iceberg::f_fields)) + collect(nested.extract()); + } + }; + for (const auto & field_var : *schema_fields) + collect(field_var.extract()); + return result; +} + +/// Iceberg name mapping (`schema.name-mapping.default`) of a type: lets readers that resolve columns by +/// field id (Spark, Trino, iceberg-java) map the columns of files written without field ids by name. +/// Mirrors `MappingUtil.create` of iceberg-java: structs map their fields, lists map `element`, maps map +/// `key` and `value`. +Poco::JSON::Array::Ptr buildNameMappingOfType(const Poco::Dynamic::Var & type_var) +{ + if (type_var.isString()) + return nullptr; + + const auto & type_obj = type_var.extract(); + const String kind = type_obj->getValue(Iceberg::f_type); + + auto make_entry = [](Int64 id, const String & name, const Poco::JSON::Array::Ptr & nested) + { + Poco::JSON::Object::Ptr entry = new Poco::JSON::Object; + entry->set("field-id", id); + Poco::JSON::Array::Ptr names = new Poco::JSON::Array; + names->add(name); + entry->set("names", names); + if (nested) + entry->set("fields", nested); + return entry; + }; + + Poco::JSON::Array::Ptr result = new Poco::JSON::Array; + if (kind == "struct") + { + for (const auto & nested_var : *type_obj->getArray(Iceberg::f_fields)) + { + const auto & nested = nested_var.extract(); + result->add(make_entry( + nested->getValue(Iceberg::f_id), nested->getValue(Iceberg::f_name), + buildNameMappingOfType(nested->get(Iceberg::f_type)))); + } + } + else if (kind == "list") + { + result->add(make_entry( + type_obj->getValue(Iceberg::f_element_id), "element", buildNameMappingOfType(type_obj->get(Iceberg::f_element)))); + } + else if (kind == "map") + { + result->add(make_entry(type_obj->getValue(Iceberg::f_key_id), "key", buildNameMappingOfType(type_obj->get(Iceberg::f_key)))); + result->add(make_entry( + type_obj->getValue(Iceberg::f_value_id), "value", buildNameMappingOfType(type_obj->get(Iceberg::f_value)))); + } + else + { + return nullptr; + } + return result; +} + +constexpr auto name_mapping_property = "schema.name-mapping.default"; + +/// Registered files are written by other tools and usually carry no Iceberg field ids. Without a name +/// mapping, readers that resolve columns by id read them as NULLs or by position. Like `add_files` of +/// iceberg-java, set the default name mapping from the current schema unless the table already has one. +/// An existing mapping is kept as is: keeping it current with schema changes is the job of the schema +/// update (as `UpdateSchema` does in iceberg-java), not of `add_files`. +/// Returns the value of the property if it was set; `metadata` is updated too. +std::optional updateDefaultNameMapping(Poco::JSON::Object::Ptr metadata, const Poco::JSON::Object::Ptr & current_schema) +{ + Poco::JSON::Object::Ptr properties; + if (metadata->has(Iceberg::f_properties) && !metadata->isNull(Iceberg::f_properties)) + properties = metadata->getObject(Iceberg::f_properties); + if (!properties) + { + properties = new Poco::JSON::Object; + metadata->set(Iceberg::f_properties, properties); + } + if (properties->has(name_mapping_property)) + return std::nullopt; + + Poco::JSON::Object::Ptr schema_as_struct = new Poco::JSON::Object; + schema_as_struct->set(Iceberg::f_type, "struct"); + schema_as_struct->set(Iceberg::f_fields, current_schema->getArray(Iceberg::f_fields)); + String value = stringifyJSON(buildNameMappingOfType(Poco::Dynamic::Var(schema_as_struct)), /* indent */ 0); + properties->set(name_mapping_property, value); + return value; +} + +/// Expand trailing-slash paths into concrete *.parquet files below them. +std::vector expandPaths( + const std::vector & raw_paths, + const IcebergPathResolver & resolver, + const ObjectStoragePtr & object_storage, + LoggerPtr log) +{ + std::vector result; + for (const auto & raw : raw_paths) + { + checkPathUnderTableLocation(raw, resolver.getTableLocation()); + if (!raw.ends_with('/')) + { + result.push_back(raw); + continue; + } + const size_t files_before = result.size(); + RelativePathsWithMetadata children; + object_storage->listObjects(resolver.resolve(IcebergPathFromMetadata::deserialize(raw)), children, /* max_keys */ 0); + for (const auto & child : children) + { + if (!endsWithParquet(child->relative_path) || hasHiddenPathComponent(child->relative_path)) + continue; + /// Listed keys are storage paths; convert to the metadata (location-prefixed) form + /// that native writes register in manifests. + String path = resolver.reverseResolve(child->relative_path).serialize(); + checkPathUnderTableLocation(path, resolver.getTableLocation()); + result.push_back(std::move(path)); + } + /// Registering nothing would commit an empty snapshot; a mistyped directory is the likely cause. + if (result.size() == files_before) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "add_files: directory '{}' contains no parquet files", raw); + LOG_DEBUG(log, "add_files: path {} expanded to {} parquet file(s)", raw, result.size() - files_before); + } + return result; +} + +Int64 floorDiv(Int64 a, Int64 b) +{ + Int64 q = a / b; + if ((a % b != 0) && ((a < 0) != (b < 0))) + --q; + return q; +} + +/// Howard Hinnant's civil-from-days algorithm. +void civilFromDays(Int64 z, Int64 & y, unsigned & m, unsigned & d) +{ + z += 719468; + Int64 era = floorDiv(z, 146097); + unsigned doe = static_cast(z - era * 146097); + unsigned yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; + Int64 year = static_cast(yoe) + era * 400; + unsigned doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + unsigned mp = (5 * doy + 2) / 153; + d = doy - (153 * mp + 2) / 5 + 1; + m = mp + (mp < 10 ? 3 : static_cast(-9)); + y = year + (m <= 2 ? 1 : 0); +} + +enum class SourceValueKind +{ + Integer, /// int32 / int64 (also date days and timestamp micros) + String, + Unsupported +}; + +/// Decoded min/max of one parquet column across all row groups, in encoded-bytes form +/// for equality and in decoded form for transforms. +struct ColumnBounds +{ + bool has_bounds = false; + /// Some row group with non-null values has no min/max, so the bounds do not cover every row. + bool incomplete = false; + /// For Integer bounds: whether the parquet physical type is INT32 (4-byte LE serialization) + /// rather than INT64 (8-byte). Decided by the physical type, never by the value range. + bool int_is_32bit = false; + Int64 min_int = 0; + Int64 max_int = 0; + String min_str; + String max_str; + SourceValueKind kind = SourceValueKind::Unsupported; +}; + +template +T decodeLE(const String & bytes) +{ + T value = 0; + memcpy(&value, bytes.data(), std::min(bytes.size(), sizeof(T))); + return value; +} + +/// Statistics of one data file derived from its parquet footer. +struct AnalyzedFile +{ + String registration_path; /// path form stored in the manifest + String storage_path; /// resolved path used for reads + IcebergSerializedFileStats stats; + /// Has an entry for every table column present in the file, with or without statistics. + std::unordered_map bounds_by_field_id; + /// Field ids whose null count is known for every row group of the file. + std::unordered_map complete_null_counts; + /// Field ids whose parquet column cannot hold nulls (it and all its parents are `REQUIRED`). + std::unordered_set non_nullable_field_ids; +}; + +/// Check that the parquet column has exactly the representation that the Iceberg type requires, +/// so that its statistics can be used as Iceberg bounds and partition values as they are. +/// Other representations (e.g. decimals, millisecond or nanosecond timestamps, INT96, widening +/// INT32 to long) are rejected rather than converted. +void checkParquetColumnType( + const parquet::ColumnDescriptor & column, const String & iceberg_type, const String & registration_path) +{ + const auto physical_type = column.physical_type(); + const auto & logical_type = column.logical_type(); + const bool no_logical_type = !logical_type || logical_type->is_none(); + + auto is_int_of_width = [&](int bit_width) + { + if (no_logical_type) + return true; + if (!logical_type->is_int()) + return false; + const auto & int_type = static_cast(*logical_type); + return int_type.bit_width() == bit_width && int_type.is_signed(); + }; + + bool matches = false; + if (iceberg_type == "int") + matches = physical_type == parquet::Type::INT32 && is_int_of_width(32); + else if (iceberg_type == "long") + matches = physical_type == parquet::Type::INT64 && is_int_of_width(64); + else if (iceberg_type == "string") + matches = physical_type == parquet::Type::BYTE_ARRAY && (no_logical_type || logical_type->is_string()); + else if (iceberg_type == "date") + matches = physical_type == parquet::Type::INT32 && logical_type && logical_type->is_date(); + else if (iceberg_type == "timestamp" || iceberg_type == "timestamptz") + matches = physical_type == parquet::Type::INT64 && logical_type && logical_type->is_timestamp() + && static_cast(*logical_type).time_unit() == parquet::LogicalType::TimeUnit::MICROS; + else if (iceberg_type == "boolean") + matches = physical_type == parquet::Type::BOOLEAN; + else if (iceberg_type == "float") + matches = physical_type == parquet::Type::FLOAT; + else if (iceberg_type == "double") + matches = physical_type == parquet::Type::DOUBLE; + else + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, + "add_files: column '{}' of file {} has Iceberg type '{}', which add_files does not support yet", + column.path()->ToDotString(), registration_path, iceberg_type); + + if (!matches) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "add_files: column '{}' of file {} is stored as parquet type {} ({}), which does not match the Iceberg type '{}' " + "of the table column", + column.path()->ToDotString(), registration_path, parquet::TypeToString(physical_type), + logical_type ? logical_type->ToString() : "no logical type", iceberg_type); +} + +ColumnBounds aggregateColumnBounds( + const parquet::ColumnChunkMetaData & column, + ColumnBounds acc) +{ + std::shared_ptr stats = column.is_stats_set() ? column.statistics() : nullptr; + if (!stats || !stats->HasMinMax()) + { + /// Only a row group whose values are all null may lack min/max. + if (!stats || !stats->HasNullCount() || stats->null_count() != column.num_values()) + acc.incomplete = true; + return acc; + } + + String encoded_min = stats->EncodeMin(); + String encoded_max = stats->EncodeMax(); + auto physical_type = stats->physical_type(); + + SourceValueKind kind = SourceValueKind::Unsupported; + if (physical_type == parquet::Type::INT32 || physical_type == parquet::Type::INT64) + kind = SourceValueKind::Integer; + else if (physical_type == parquet::Type::BYTE_ARRAY || physical_type == parquet::Type::FIXED_LEN_BYTE_ARRAY) + kind = SourceValueKind::String; + + if (kind == SourceValueKind::Unsupported) + return acc; + + if (!acc.has_bounds) + { + acc.kind = kind; + if (kind == SourceValueKind::Integer) + { + acc.int_is_32bit = physical_type == parquet::Type::INT32; + acc.min_int = physical_type == parquet::Type::INT32 ? Int64(decodeLE(encoded_min)) : decodeLE(encoded_min); + acc.max_int = physical_type == parquet::Type::INT32 ? Int64(decodeLE(encoded_max)) : decodeLE(encoded_max); + } + else + { + acc.min_str = encoded_min; + acc.max_str = encoded_max; + } + acc.has_bounds = true; + return acc; + } + + if (kind != acc.kind) + throw Exception(ErrorCodes::LOGICAL_ERROR, "add_files: statistics of one column have different physical types"); + + if (kind == SourceValueKind::Integer) + { + Int64 this_min = physical_type == parquet::Type::INT32 ? Int64(decodeLE(encoded_min)) : decodeLE(encoded_min); + Int64 this_max = physical_type == parquet::Type::INT32 ? Int64(decodeLE(encoded_max)) : decodeLE(encoded_max); + acc.min_int = std::min(acc.min_int, this_min); + acc.max_int = std::max(acc.max_int, this_max); + } + else + { + acc.min_str = std::min(acc.min_str, encoded_min); + acc.max_str = std::max(acc.max_str, encoded_max); + } + return acc; +} + +/// Read the parquet footer of one file and derive its Iceberg statistics and per-column bounds. +AnalyzedFile analyzeFile( + const String & registration_path, + const IcebergPathResolver & resolver, + const ObjectStoragePtr & object_storage, + const std::unordered_map & column_path_to_field_id, + const std::unordered_map & iceberg_type_by_field_id, + const ContextPtr & context, + const FormatSettings & format_settings) +{ + AnalyzedFile result; + result.registration_path = registration_path; + + IcebergPathFromMetadata metadata_path = IcebergPathFromMetadata::deserialize(registration_path); + result.storage_path = resolver.resolve(metadata_path); + + auto object_metadata = object_storage->tryGetObjectMetadata(result.storage_path, /* with_tags */ false); + if (!object_metadata.has_value()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "add_files: file {} does not exist", registration_path); + result.stats.file_size_in_bytes = static_cast(object_metadata->size_bytes); + + std::shared_ptr parquet_metadata; + try + { + std::atomic is_stopped = 0; + auto buffer = object_storage->readObject(StoredObject(result.storage_path), context->getReadSettings()); + auto arrow_file = asArrowFile(*buffer, format_settings, is_stopped, "Parquet", PARQUET_MAGIC_BYTES, /* avoid_buffering */ true); + parquet_metadata = parquet::ReadMetaData(arrow_file); + } + catch (...) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "add_files: failed to read the parquet footer of {} ({}). The file may be truncated " + "or still being written by another process; registering such a file is the caller's responsibility. " + "Original error: {}", + registration_path, result.storage_path, getCurrentExceptionMessage(false)); + } + + result.stats.record_count = parquet_metadata->num_rows(); + + /// Validate the type of every table column present in the file. Columns whose Iceberg type is not + /// known here (leaves of lists and maps) are registered without bounds. + std::unordered_set columns_with_bounds; + const auto * parquet_schema = parquet_metadata->schema(); + for (int col = 0; col < parquet_schema->num_columns(); ++col) + { + const auto * descriptor = parquet_schema->Column(col); + auto field_it = column_path_to_field_id.find(descriptor->path()->ToDotString()); + if (field_it == column_path_to_field_id.end()) + continue; + auto type_it = iceberg_type_by_field_id.find(field_it->second); + if (type_it == iceberg_type_by_field_id.end()) + continue; + checkParquetColumnType(*descriptor, type_it->second, registration_path); + columns_with_bounds.insert(static_cast(field_it->second)); + if (descriptor->max_definition_level() == 0) + result.non_nullable_field_ids.insert(static_cast(field_it->second)); + } + + std::unordered_map column_sizes; + std::unordered_map null_value_counts; + std::unordered_set incomplete_null_counts; + std::unordered_map bounds; + + for (int rg = 0; rg < parquet_metadata->num_row_groups(); ++rg) + { + auto row_group = parquet_metadata->RowGroup(rg); + for (int col = 0; col < row_group->num_columns(); ++col) + { + auto column = row_group->ColumnChunk(col); + String dot_path = column->path_in_schema()->ToDotString(); + auto field_it = column_path_to_field_id.find(dot_path); + if (field_it == column_path_to_field_id.end()) + continue; + Int32 field_id = static_cast(field_it->second); + + column_sizes[field_id] += column->total_compressed_size(); + std::shared_ptr stats = column->is_stats_set() ? column->statistics() : nullptr; + if (stats && stats->HasNullCount()) + null_value_counts[field_id] += stats->null_count(); + else + incomplete_null_counts.insert(field_id); + + if (columns_with_bounds.contains(field_id)) + bounds[field_id] = aggregateColumnBounds(*column, std::move(bounds[field_id])); + else + bounds.try_emplace(field_id); + } + } + + for (const auto & [field_id, size] : column_sizes) + result.stats.column_sizes.emplace_back(field_id, size); + for (const auto & [field_id, count] : null_value_counts) + { + if (incomplete_null_counts.contains(field_id)) + continue; + result.stats.null_value_counts.emplace_back(field_id, count); + result.complete_null_counts.emplace(field_id, count); + } + for (auto & [field_id, bound] : bounds) + { + if (bound.incomplete) + bound.has_bounds = false; + if (!bound.has_bounds) + continue; + if (bound.kind == SourceValueKind::Integer) + { + /// Pass the parquet plain encoding through: both parquet statistics and Iceberg + /// bounds store fixed-width numerics little-endian, with the width of the parquet + /// physical type (INT32 -> 4 bytes, INT64 -> 8 bytes). + std::vector lower_bytes; + std::vector upper_bytes; + if (bound.int_is_32bit) + { + Int32 lo = static_cast(bound.min_int); + Int32 hi = static_cast(bound.max_int); + lower_bytes.resize(sizeof(Int32)); + upper_bytes.resize(sizeof(Int32)); + memcpy(lower_bytes.data(), &lo, sizeof(Int32)); + memcpy(upper_bytes.data(), &hi, sizeof(Int32)); + } + else + { + Int64 lo = bound.min_int; + Int64 hi = bound.max_int; + lower_bytes.resize(sizeof(Int64)); + upper_bytes.resize(sizeof(Int64)); + memcpy(lower_bytes.data(), &lo, sizeof(Int64)); + memcpy(upper_bytes.data(), &hi, sizeof(Int64)); + } + result.stats.lower_bounds.emplace_back(field_id, std::move(lower_bytes)); + result.stats.upper_bounds.emplace_back(field_id, std::move(upper_bytes)); + } + else + { + std::vector lower_bytes(bound.min_str.begin(), bound.min_str.end()); + std::vector upper_bytes(bound.max_str.begin(), bound.max_str.end()); + result.stats.lower_bounds.emplace_back(field_id, std::move(lower_bytes)); + result.stats.upper_bounds.emplace_back(field_id, std::move(upper_bytes)); + } + } + + result.bounds_by_field_id = std::move(bounds); + return result; +} + +/// The value one partition field takes for a file, proven identical for every row in the file. +struct PartitionFieldValue +{ + String column_name; + Field value; /// Null when the transform is void or the column has only nulls + DataTypePtr type; /// ClickHouse type used for manifest tuple serialization +}; + +/// Derive the partition tuple of one file from its per-column bounds and prove that every row +/// in the file belongs to that single partition. Fails with an error naming the file and the +/// offending column when a file spans partitions. +PartitionFieldValue derivePartitionField( + const Poco::JSON::Object::Ptr & spec_field, + const AnalyzedFile & file, + const std::unordered_map & source_columns, + const std::unordered_map & iceberg_type_by_field_id) +{ + Int64 source_id = spec_field->getValue(Iceberg::f_source_id); + String transform = spec_field->getValue(Iceberg::f_transform); + String field_name = spec_field->getValue(Iceberg::f_name); + + PartitionFieldValue result; + result.column_name = field_name; + + auto source_it = source_columns.find(source_id); + if (source_it == source_columns.end()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "add_files: partition field '{}' refers to source id {} which is not in the current schema", field_name, source_id); + const NameAndTypePair & source_column = source_it->second; + + if (transform == "void") + { + result.value = Null(); + result.type = source_column.type; + return result; + } + + /// Parse bucket[N] / truncate[N]. + auto bracket = transform.find('['); + String transform_name = bracket == String::npos ? transform : transform.substr(0, bracket); + Int64 transform_argument = 0; + if (bracket != String::npos) + { + const char * begin = transform.data() + bracket + 1; + const char * end = transform.data() + transform.size() - 1; + auto [ptr, ec] = std::from_chars(begin, end, transform_argument); + if (ec != std::errc() || ptr != end || transform.back() != ']' || transform_argument <= 0) + throw Exception(ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "add_files: partition field '{}' has malformed transform '{}'", field_name, transform); + } + + if (transform_name == "bucket") + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, + "add_files: partition field '{}' uses the bucket transform, which cannot be validated from " + "file statistics alone (two different values may share a bucket with a third outside it). " + "Files for bucket-partitioned tables are not supported yet", field_name); + + auto bounds_it = file.bounds_by_field_id.find(static_cast(source_id)); + if (bounds_it == file.bounds_by_field_id.end() || !bounds_it->second.has_bounds) + { + /// The partition value is null only if every row has null in the source column: the column is + /// absent from the file (allowed by `allow_missing_columns`), or the footer proves that all of its + /// values are null. Missing statistics alone prove nothing (writers drop them, e.g. for long values). + if (bounds_it != file.bounds_by_field_id.end()) + { + auto null_count_it = file.complete_null_counts.find(static_cast(source_id)); + if (null_count_it == file.complete_null_counts.end() || null_count_it->second != file.stats.record_count) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "add_files: cannot derive the partition value of file {}: the parquet footer has no complete " + "min/max statistics for the partition source column '{}'", + file.registration_path, source_column.name); + } + result.value = Null(); + result.type = transform_name == "identity" || transform_name == "truncate" ? source_column.type : DataTypePtr(std::make_shared()); + return result; + } + const ColumnBounds & bounds = bounds_it->second; + + auto iceberg_type_it = iceberg_type_by_field_id.find(source_id); + const String iceberg_type = iceberg_type_it != iceberg_type_by_field_id.end() ? iceberg_type_it->second : ""; + bool is_timestamp = iceberg_type.starts_with("timestamp"); + bool is_date = iceberg_type == "date"; + + if (transform_name == "identity") + { + bool equal = bounds.kind == SourceValueKind::Integer ? bounds.min_int == bounds.max_int : bounds.min_str == bounds.max_str; + if (!equal) + throw Exception(ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "add_files: file {} spans more than one partition: column '{}' has distinct min and max values", + file.registration_path, source_column.name); + if (bounds.kind == SourceValueKind::Integer) + { + if (is_date) + { + /// Mirrors the native writer: identity Date partition values are the raw day number, + /// promoted to UInt64 for `Date` (ColumnVector) and Int64 for `Date32`. + if (WhichDataType(removeNullable(source_column.type)).isDate32()) + result.value = Field(Int64(bounds.min_int)); + else if (bounds.min_int >= 0 && bounds.min_int <= std::numeric_limits::max()) + result.value = Field(UInt64(bounds.min_int)); + else + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "add_files: date {} of column '{}' in file {} is out of the range of the table column type {}", + bounds.min_int, source_column.name, file.registration_path, source_column.type->getName()); + } + else if (is_timestamp) + { + /// Mirrors the native writer: a DateTime64 partition column carries its raw + /// decimal value (scale <= 6, i.e. microseconds for Iceberg timestamp types). + auto scale = getDecimalScale(*removeNullable(source_column.type)); + result.value = Field(DecimalField(Decimal64(bounds.min_int), scale)); + } + else + { + result.value = Field(bounds.min_int); + } + } + else + { + result.value = Field(bounds.min_str); + } + result.type = source_column.type; + return result; + } + + if (bounds.kind != SourceValueKind::Integer && transform_name != "truncate") + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "add_files: partition field '{}' applies transform '{}' to a non-numeric column '{}'", + field_name, transform, source_column.name); + + auto days_of = [&](Int64 v) { return is_timestamp ? floorDiv(v, 86400000000LL) : v; }; + auto year_of = [&](Int64 v) + { + Int64 y; + unsigned m, d; + civilFromDays(days_of(v), y, m, d); + return y - 1970; + }; + auto month_of = [&](Int64 v) + { + Int64 y; + unsigned m, d; + civilFromDays(days_of(v), y, m, d); + return (y - 1970) * 12 + static_cast(m) - 1; + }; + + Int64 lo = bounds.min_int; + Int64 hi = bounds.max_int; + Int64 lo_transformed = 0; + Int64 hi_transformed = 0; + + if (transform_name == "year") + { + lo_transformed = year_of(lo); + hi_transformed = year_of(hi); + } + else if (transform_name == "month") + { + lo_transformed = month_of(lo); + hi_transformed = month_of(hi); + } + else if (transform_name == "day" || transform_name == "days" || transform_name == "date" || transform_name == "dates") + { + lo_transformed = days_of(lo); + hi_transformed = days_of(hi); + } + else if (transform_name == "hour" || transform_name == "hours") + { + lo_transformed = floorDiv(lo, 3600000000LL); + hi_transformed = floorDiv(hi, 3600000000LL); + } + else if (transform_name == "truncate") + { + if (bounds.kind == SourceValueKind::String) + { + String lo_trunc = truncateToCodePoints(bounds.min_str, static_cast(transform_argument)); + String hi_trunc = truncateToCodePoints(bounds.max_str, static_cast(transform_argument)); + if (lo_trunc != hi_trunc) + throw Exception(ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "add_files: file {} spans more than one partition: truncated values of column '{}' differ", + file.registration_path, source_column.name); + result.value = Field(lo_trunc); + result.type = source_column.type; + return result; + } + lo_transformed = lo - ((lo % transform_argument + transform_argument) % transform_argument); + hi_transformed = hi - ((hi % transform_argument + transform_argument) % transform_argument); + } + else + { + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, + "add_files: partition field '{}' uses unsupported transform '{}'", field_name, transform); + } + + if (lo_transformed != hi_transformed) + throw Exception(ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "add_files: file {} spans more than one partition: column '{}' maps to different '{}' buckets " + "(min {} -> {}, max {} -> {})", + file.registration_path, source_column.name, transform_name, lo, lo_transformed, hi, hi_transformed); + + result.value = Field(lo_transformed); + result.type = transform_name == "truncate" ? source_column.type : DataTypePtr(std::make_shared()); + return result; +} + +String partitionKeyToString(const std::vector & values) +{ + WriteBufferFromOwnString buf; + for (const auto & value : values) + writeString(applyVisitor(FieldVisitorToString(), value), buf); + return buf.str(); +} + +/// Collect the paths of all data files live in the current snapshot, for duplicate detection. +std::unordered_set listLiveDataFiles( + Poco::JSON::Object::Ptr metadata, + const IcebergPathResolver & resolver, + const ObjectStoragePtr & object_storage, + const FormatSettings & format_settings) +{ + std::unordered_set live_paths; + String manifest_list = getCurrentManifestListPath(metadata); + if (manifest_list.empty()) + return live_paths; + + auto read_avro = [&](const String & path) + { + auto buffer = object_storage->readObject( + StoredObject(resolver.resolve(IcebergPathFromMetadata::deserialize(path))), ReadSettings{}); + return AvroForIcebergDeserializer(std::move(buffer), IcebergPathFromMetadata::deserialize(path), format_settings); + }; + + auto manifest_list_reader = read_avro(manifest_list); + for (size_t i = 0; i < manifest_list_reader.rows(); ++i) + { + /// Only data manifests (content == 0) hold data files. + if (manifest_list_reader.getValueFromRowByName(i, Iceberg::f_content, TypeIndex::Int32).safeGet() != 0) + continue; + String manifest_path = manifest_list_reader.getValueFromRowByName(i, Iceberg::f_manifest_path).safeGet(); + auto manifest_reader = read_avro(manifest_path); + for (size_t j = 0; j < manifest_reader.rows(); ++j) + { + auto entry = manifest_reader.getParsedManifestFileEntry(j); + if (entry->status != ManifestEntryStatus::DELETED) + live_paths.insert(entry->file_path_key.serialize()); + } + } + return live_paths; +} + +/// Whether the table metadata that `catalog` currently points to contains a snapshot with `snapshot_id`. +/// Resolves the "commit state unknown" case. Throws if the catalog or the metadata file cannot be read, +/// so that the caller never treats an unknown state as "not committed". +bool isSnapshotInCatalogMetadata( + const std::shared_ptr & catalog, + const String & table_name, + Int64 snapshot_id, + const PersistentTableComponents & persistent_components, + const DataLakeStorageSettings & data_lake_settings, + const ObjectStoragePtr & object_storage, + const ContextPtr & context, + LoggerPtr log) +{ + const auto metadata_file_info = getLatestMetadataFileAndVersionWithCatalog( + object_storage, + catalog, + table_name, + persistent_components.table_path, + data_lake_settings, + persistent_components.metadata_cache, + context, + log.get(), + persistent_components.table_uuid, + persistent_components.metadata_compression_method); + auto metadata = getMetadataJSONObject( + metadata_file_info.path, + object_storage, + persistent_components.metadata_cache, + context, + log, + metadata_file_info.compression_method, + persistent_components.table_uuid); + + if (!metadata->has(Iceberg::f_snapshots)) + return false; + for (const auto & snapshot_var : *metadata->getArray(Iceberg::f_snapshots)) + { + if (snapshot_var.extract()->getValue(Iceberg::f_metadata_snapshot_id) == snapshot_id) + return true; + } + return false; +} + +Pipe resultToPipe(const AddFilesResult & result) +{ + Block header{ + ColumnWithTypeAndName(std::make_shared(), "metric_name"), + ColumnWithTypeAndName(std::make_shared(), "metric_value"), + }; + + MutableColumns columns = header.cloneEmptyColumns(); + auto add = [&](const char * name, Int64 value) + { + columns[0]->insert(String(name)); + columns[1]->insert(value); + }; + + add("files_registered", result.files_registered); + add("rows_registered", result.rows_registered); + add("bytes_registered", result.bytes_registered); + add("partitions_touched", result.partitions_touched); + add("manifests_written", result.manifests_written); + + const size_t rows = columns[0]->size(); + Chunk chunk(std::move(columns), rows); + return Pipe(std::make_shared(std::make_shared(std::move(header)), std::move(chunk))); +} + +} + +AddFilesResult addFiles( + const std::vector & raw_paths, + bool check_duplicate_files, + bool allow_missing_columns, + ContextPtr context, + ObjectStoragePtr object_storage, + const DataLakeStorageSettings & data_lake_settings, + const PersistentTableComponents & persistent_components, + const String & write_format, + std::shared_ptr catalog, + const String & table_name, + ExternalStorageCache & external_storages) +{ + if (Poco::toLower(write_format) != "parquet") + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, + "add_files currently supports only Parquet data files, the table's write format is '{}'", write_format); + + const auto & resolver = persistent_components.path_resolver; + LoggerPtr log = getLogger("IcebergAddFiles"); + FormatSettings format_settings = getFormatSettings(context); + + FileNamesGenerator filename_generator( + resolver.getTableLocation(), + (catalog != nullptr && catalog->isTransactional()), + persistent_components.metadata_compression_method, + write_format); + + /// With a catalog, the catalog names the current metadata file. Listing the storage instead may pick the + /// metadata of another table that shares the location, or miss a commit that is visible only in the catalog. + auto get_current_metadata_file = [&]() + { + return getLatestMetadataFileAndVersionWithCatalog( + object_storage, + catalog, + table_name, + persistent_components.table_path, + data_lake_settings, + persistent_components.metadata_cache, + context, + log.get(), + persistent_components.table_uuid, + persistent_components.metadata_compression_method, + /* ignore_explicit_metadata_file_path */ false); + }; + + auto [last_version, metadata_path, compression_method] = get_current_metadata_file(); + filename_generator.setVersion(last_version + 1); + + Poco::JSON::Object::Ptr metadata = getMetadataJSONObject( + metadata_path, + object_storage, + persistent_components.metadata_cache, + context, + log, + compression_method, + persistent_components.table_uuid); + + const Int64 original_schema_id = metadata->getValue(Iceberg::f_current_schema_id); + const Int64 partition_spec_id = metadata->getValue(Iceberg::f_default_spec_id); + Poco::JSON::Object::Ptr partition_spec = lookupPartitionSpec(metadata, partition_spec_id); + Poco::JSON::Object::Ptr current_schema = lookupSchema(metadata, original_schema_id); + Poco::JSON::Array::Ptr schema_fields = current_schema->getArray(Iceberg::f_fields); + + /// Column path -> field id and back, over all (including nested) leaf fields. + std::unordered_map column_path_to_field_id = IcebergSchemaProcessor::traverseSchema(schema_fields); + std::unordered_map field_id_to_column_path; + for (const auto & [path, id] : column_path_to_field_id) + field_id_to_column_path.emplace(id, path); + + /// Iceberg primitive type per field id (for date/timestamp transform scaling). + auto iceberg_type_by_field_id = collectIcebergTypesByFieldId(schema_fields); + + /// Source columns for partition fields, from the ClickHouse schema (top-level fields only). + /// The schema processor is populated lazily (e.g. by reads or native writes); on a freshly + /// created table this add_files call may be the first operation, so register the current + /// schema ourselves first. + if (!persistent_components.schema_processor->hasClickHouseTableSchemaById(static_cast(original_schema_id))) + persistent_components.schema_processor->addIcebergTableSchema(current_schema, context); + auto clickhouse_schema = persistent_components.schema_processor->getClickHouseTableSchemaById(static_cast(original_schema_id)); + Block sample_block; + for (const auto & name_and_type : *clickhouse_schema) + sample_block.insert(ColumnWithTypeAndName(name_and_type.type, name_and_type.name)); + SharedHeader shared_sample_block = std::make_shared(sample_block); + + std::unordered_map source_columns; + Poco::JSON::Array::Ptr spec_fields = partition_spec->getArray(Iceberg::f_fields); + for (const auto & spec_field_var : *spec_fields) + { + auto spec_field = spec_field_var.extract(); + Int64 source_id = spec_field->getValue(Iceberg::f_source_id); + auto path_it = field_id_to_column_path.find(source_id); + if (path_it == field_id_to_column_path.end()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "add_files: partition spec refers to source id {} which is not in the current schema", source_id); + if (path_it->second.find('.') != String::npos) + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, + "add_files: partition source column '{}' is nested; only top-level partition columns are supported", path_it->second); + auto type_it = std::find_if(clickhouse_schema->begin(), clickhouse_schema->end(), + [&](const NameAndTypePair & nt) { return nt.name == path_it->second; }); + if (type_it == clickhouse_schema->end()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Column {} not found in ClickHouse schema", path_it->second); + source_columns.emplace(source_id, *type_it); + } + + /// Expand trailing-slash paths. Registration paths are kept exactly as given (or produced + /// by expansion): they must be in metadata form, i.e. prefixed by the table location, which + /// is the same form native writes register in manifests. + std::vector paths = expandPaths(raw_paths, resolver, object_storage, log); + for (const auto & path : paths) + { + if (!endsWithParquet(path)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "add_files: only .parquet files can be registered, got '{}'", path); + } + + /// Read every file once: footer statistics + bounds. + std::vector analyzed_files; + analyzed_files.reserve(paths.size()); + for (const auto & path : paths) + analyzed_files.push_back(analyzeFile(path, resolver, object_storage, column_path_to_field_id, iceberg_type_by_field_id, context, format_settings)); + + /// Validate the file's columns against the table schema. A missing column reads as NULL, which a + /// required field must never return, so `allow_missing_columns` applies to optional fields only. + const auto required_field_ids = collectRequiredFieldIds(schema_fields); + for (const auto & file : analyzed_files) + { + for (const auto & [field_path, field_id] : column_path_to_field_id) + { + if (!iceberg_type_by_field_id.contains(field_id)) + continue; + if (file.bounds_by_field_id.contains(static_cast(field_id))) + { + /// A required field must have a value in every row. Readers return the type default or fail + /// for nulls in it, so the file must prove that the column has none. + if (required_field_ids.contains(field_id) && !file.non_nullable_field_ids.contains(static_cast(field_id))) + { + auto null_count_it = file.complete_null_counts.find(static_cast(field_id)); + if (null_count_it == file.complete_null_counts.end()) + throw Exception(ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "add_files: column '{}' of file {} is required in the table schema, but the file may contain " + "nulls in it: the column is optional in the file and the parquet footer has no complete null counts", + field_path, file.registration_path); + if (null_count_it->second != 0) + throw Exception(ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "add_files: column '{}' of file {} is required in the table schema, but the file contains {} null(s) in it", + field_path, file.registration_path, null_count_it->second); + } + continue; + } + if (required_field_ids.contains(field_id)) + throw Exception(ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "add_files: file {} lacks column '{}', which is required in the table schema", + file.registration_path, field_path); + if (!allow_missing_columns) + throw Exception(ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "add_files: file {} lacks column '{}' present in the table schema; " + "pass allow_missing_columns = 1 to register it anyway (its values will read as NULL)", + file.registration_path, field_path); + } + } + + /// Derive (and validate) the partition tuple of every file, then group files by tuple. + struct PartitionGroup + { + std::vector partition_columns; + std::vector partition_values; + DataTypes partition_types; + std::vector file_indices; + }; + std::map groups; + for (size_t i = 0; i < analyzed_files.size(); ++i) + { + std::vector tuple; + std::vector names; + DataTypes types; + for (const auto & spec_field_var : *spec_fields) + { + auto field_value = derivePartitionField( + spec_field_var.extract(), analyzed_files[i], source_columns, iceberg_type_by_field_id); + names.push_back(field_value.column_name); + tuple.push_back(field_value.value); + types.push_back(field_value.type); + } + String key = partitionKeyToString(tuple); + auto [it, inserted] = groups.try_emplace(key); + if (inserted) + { + it->second.partition_columns = names; + it->second.partition_values = tuple; + it->second.partition_types = types; + } + it->second.file_indices.push_back(i); + } + + /// Duplicate detection against the current snapshot and inside the input set. + if (check_duplicate_files) + { + std::unordered_set seen; + for (const auto & file : analyzed_files) + { + if (!seen.insert(file.registration_path).second) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "add_files: file '{}' is listed twice in the input", file.registration_path); + } + } + + /// Repeated on every retry: a concurrent commit may have registered the same files. + auto check_files_not_registered = [&]() + { + if (!check_duplicate_files) + return; + auto live_files = listLiveDataFiles(metadata, resolver, object_storage, format_settings); + for (const auto & file : analyzed_files) + { + if (live_files.contains(file.registration_path)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "add_files: file '{}' is already registered in the current snapshot", file.registration_path); + } + }; + check_files_not_registered(); + + const bool catalog_is_transactional = catalog && catalog->isTransactional(); + + /// Commit: one manifest per partition tuple, one manifest list, one append snapshot. + AddFilesResult result; + result.partitions_touched = static_cast(groups.size()); + + size_t retry = 0; + static constexpr size_t max_retries = 100; + while (true) + { + auto metadata_info = filename_generator.generateMetadataPathWithInfo(); + + Int64 parent_snapshot = -1; + if (metadata->has(Iceberg::f_current_snapshot_id) && !metadata->isNull(Iceberg::f_current_snapshot_id)) + parent_snapshot = metadata->getValue(Iceberg::f_current_snapshot_id); + + Int64 total_files = 0; + Int64 total_rows = 0; + Int64 total_bytes = 0; + for (const auto & file : analyzed_files) + { + ++total_files; + total_rows += file.stats.record_count; + total_bytes += file.stats.file_size_in_bytes; + } + + auto [new_snapshot, manifest_list_path] = MetadataGenerator(metadata).generateNextMetadata( + filename_generator, metadata_info.path, parent_snapshot, + total_files, total_rows, total_bytes, static_cast(groups.size()), + /* added_delete_files */ 0, /* num_deleted_rows */ 0); + const auto storage_manifest_list_name = resolver.resolve(manifest_list_path); + + std::vector manifest_paths; + std::vector manifest_lengths; + std::vector manifest_row_counts; + std::vector>> manifest_partition_summaries; + bool published = false; + + auto cleanup = [&](bool retry_because_of_metadata_conflict) + { + /// The new snapshot may be visible: its files must stay, even if that leaves orphans. + if (published) + { + LOG_INFO(log, "Not removing the new manifest files: the new snapshot may already be visible"); + return; + } + for (size_t i = 0; i < manifest_paths.size(); ++i) + object_storage->removeObjectIfExists(StoredObject(resolver.resolve(manifest_paths[i]))); + object_storage->removeObjectIfExists(StoredObject(storage_manifest_list_name)); + + if (retry_because_of_metadata_conflict) + { + auto [new_version, new_metadata_path, new_compression] = get_current_metadata_file(); + filename_generator.setVersion(new_version + 1); + metadata = getMetadataJSONObject( + new_metadata_path, object_storage, persistent_components.metadata_cache, + context, log, new_compression, persistent_components.table_uuid); + + if (metadata->getValue(Iceberg::f_current_schema_id) != original_schema_id) + throw Exception(ErrorCodes::METADATA_MISMATCH, + "Table schema changed during add_files; restart the command"); + if (metadata->getValue(Iceberg::f_default_spec_id) != partition_spec_id) + throw Exception(ErrorCodes::METADATA_MISMATCH, + "Table partition spec changed during add_files; restart the command"); + check_files_not_registered(); + } + }; + + try + { + /// Write one manifest per partition tuple. + for (const auto & [key, group] : groups) + { + auto manifest_path = filename_generator.generateManifestEntryName(); + auto storage_manifest_name = resolver.resolve(manifest_path); + auto buffer = object_storage->writeObject( + StoredObject(storage_manifest_name), WriteMode::Rewrite, std::nullopt, + DBMS_DEFAULT_BUFFER_SIZE, context->getWriteSettings()); + + std::vector group_paths; + std::vector group_row_counts; + std::vector group_byte_counts; + std::vector group_stats; + Int64 group_rows = 0; + for (size_t idx : group.file_indices) + { + group_paths.push_back(IcebergPathFromMetadata::deserialize(analyzed_files[idx].registration_path)); + group_row_counts.push_back(static_cast(analyzed_files[idx].stats.record_count)); + group_byte_counts.push_back(static_cast(analyzed_files[idx].stats.file_size_in_bytes)); + group_stats.push_back(analyzed_files[idx].stats); + group_rows += analyzed_files[idx].stats.record_count; + } + + generateManifestFile( + metadata, + group.partition_columns, + group.partition_values, + group.partition_types, + group_paths, + group_row_counts, + group_byte_counts, + /* data_file_statistics */ std::nullopt, + shared_sample_block, + new_snapshot, + write_format, + partition_spec, + partition_spec_id, + *buffer, + FileContentType::DATA, + /* user_defined_sequence_number */ std::nullopt, + /* user_defined_snapshot_id */ std::nullopt, + /* data_file_formats */ {}, + /* per_file_statistics */ {}, + /* data_file_sort_order_ids */ {}, + /* per_file_entry_lineage */ {}, + /* schema_to_serialize */ nullptr, + /* per_file_fresh_statistics */ nullptr, + /* per_file_stats */ group_stats); + buffer->finalize(); + + manifest_paths.push_back(manifest_path); + manifest_lengths.push_back(static_cast(buffer->count())); + manifest_row_counts.push_back(group_rows); + + std::vector> summary; + summary.reserve(group.partition_values.size()); + for (size_t i = 0; i < group.partition_values.size(); ++i) + summary.emplace_back(group.partition_values[i], group.partition_types[i]); + manifest_partition_summaries.push_back(std::move(summary)); + } + + { + auto buffer_manifest_list = object_storage->writeObject( + StoredObject(storage_manifest_list_name), WriteMode::Rewrite, std::nullopt, + DBMS_DEFAULT_BUFFER_SIZE, context->getWriteSettings()); + generateManifestList( + resolver, metadata, object_storage, external_storages, context, + manifest_paths, new_snapshot, manifest_lengths, *buffer_manifest_list, + FileContentType::DATA, /* use_previous_snapshots */ true, + /* per_entry_content_types */ {}, + /* entry_counts */ {}, + /* carry_forward_manifest_paths */ {}, + /* entry_partition_spec_ids */ {}, + /* entry_partition_summaries */ manifest_partition_summaries, + /* previous_manifest_actions */ {}, + /* entry_row_counts */ manifest_row_counts); + buffer_manifest_list->finalize(); + } + + { + /// A transactional catalog builds the new metadata itself, so the changed property is sent with the commit. + std::map properties_to_set; + if (auto name_mapping = updateDefaultNameMapping(metadata, lookupSchema(metadata, original_schema_id))) + properties_to_set.emplace(name_mapping_property, *name_mapping); + std::string json_representation = stringifyJSON(metadata, 4); + + /// A transactional catalog writes the metadata file itself, as for native inserts. + if (!catalog_is_transactional) + { + auto hint_path = filename_generator.generateVersionHint(); + if (!writeMetadataFileAndVersionHint( + resolver, metadata_info, json_representation, hint_path, object_storage, context, + data_lake_settings[DataLakeStorageSetting::iceberg_use_version_hint])) + { + cleanup(true); + if (++retry >= max_retries) + throw Exception(ErrorCodes::LIMIT_EXCEEDED, "add_files: too many metadata conflicts, giving up"); + continue; + } + /// The metadata file and version hint in storage now name the new snapshot. + published = true; + } + + if (catalog) + { + auto catalog_filename = resolver.resolveForCatalog(metadata_info.path); + const auto & [namespace_name, table] = DataLake::parseTableName(table_name); + + if (!catalog_is_transactional) + { + /// The snapshot is already visible through storage: deleting its files would break it, and a + /// retry would build on it. Keep the files and report the error. + if (!catalog->updateMetadataAndProperties( + namespace_name, table, catalog_filename, new_snapshot, properties_to_set, context->getForwardedAuthToken())) + throw Exception( + ErrorCodes::DATALAKE_DATABASE_ERROR, + "add_files: Iceberg metadata {} of table {}.{} was committed to storage, but the catalog rejected " + "the update of the table pointer. The table in storage is consistent; the catalog still names " + "the previous metadata", + catalog_filename, namespace_name, table); + } + else + { + bool updated = false; + std::exception_ptr update_exception; + try + { + updated = catalog->updateMetadataAndProperties( + namespace_name, table, catalog_filename, new_snapshot, properties_to_set, context->getForwardedAuthToken()); + } + catch (...) + { + update_exception = std::current_exception(); + } + + if (!updated) + { + /// Commit state unknown: the catalog may have applied the update although the client saw + /// a failure. Until the catalog proves otherwise, keep the files, even if the check throws. + published = true; + const Int64 new_snapshot_id = new_snapshot->getValue(Iceberg::f_metadata_snapshot_id); + if (isSnapshotInCatalogMetadata( + catalog, table_name, new_snapshot_id, persistent_components, data_lake_settings, + object_storage, context, log)) + { + LOG_INFO(log, "Catalog update for snapshot {} was reported as failed, but the catalog has applied it", + new_snapshot_id); + } + else + { + /// The update did not land: no snapshot references the new files. + published = false; + if (update_exception) + std::rethrow_exception(update_exception); + cleanup(true); + if (++retry >= max_retries) + throw Exception(ErrorCodes::LIMIT_EXCEEDED, "add_files: too many catalog conflicts, giving up"); + continue; + } + } + } + } + published = true; + } + + persistent_components.invalidateMetadataCache(); + } + catch (...) + { + if (published) + { + tryLogCurrentException(log, + "Post-publish work failed after Iceberg snapshot was committed by add_files; " + "skipping manifest cleanup to preserve the published snapshot"); + } + else + { + cleanup(false); + } + throw; + } + + result.files_registered = total_files; + result.rows_registered = total_rows; + result.bytes_registered = total_bytes; + result.manifests_written = static_cast(manifest_paths.size()); + return result; + } +} + +Pipe executeAddFiles( + const ASTPtr & args, + ContextPtr context, + ObjectStoragePtr object_storage, + const DataLakeStorageSettings & data_lake_settings, + const PersistentTableComponents & persistent_components, + const String & write_format, + std::shared_ptr catalog, + const String & table_name, + ExternalStorageCache & external_storages) +{ + auto parsed = makeSchema().parse(args); + std::vector paths = parsePathsArgument(parsed.get("paths")); + bool check_duplicates = parsed.getAs("check_duplicate_files") != 0; + bool allow_missing = parsed.getAs("allow_missing_columns") != 0; + + auto result = addFiles( + paths, check_duplicates, allow_missing, + context, object_storage, data_lake_settings, persistent_components, + write_format, catalog, table_name, external_storages); + + return resultToPipe(result); +} + +} +} + +#endif diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/AddFilesExecute.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/AddFilesExecute.h new file mode 100644 index 0000000000000..0c3248947cf0e --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/AddFilesExecute.h @@ -0,0 +1,58 @@ +#pragma once + +#include "config.h" + +#if USE_AVRO + +#include +#include +#include +#include +#include +#include +#include + +namespace DB::Iceberg +{ + +/// Result of an add_files execution, surfaced as (metric_name, metric_value) rows. +struct AddFilesResult +{ + Int64 files_registered = 0; + Int64 rows_registered = 0; + Int64 bytes_registered = 0; + Int64 partitions_touched = 0; + Int64 manifests_written = 0; +}; + +/// Register existing Parquet files as a new append snapshot, like iceberg-java's +/// `add_files`. Each file is read once (footer only) to derive its record count, +/// column sizes, null counts and bounds, and to prove that it belongs to a single +/// partition of the table's current default partition spec. +AddFilesResult addFiles( + const std::vector & paths, + bool check_duplicate_files, + bool allow_missing_columns, + ContextPtr context, + ObjectStoragePtr object_storage, + const DataLakeStorageSettings & data_lake_settings, + const PersistentTableComponents & persistent_table_components, + const String & write_format, + std::shared_ptr catalog, + const String & table_name, + ExternalStorageCache & external_storages); + +Pipe executeAddFiles( + const ASTPtr & args, + ContextPtr context, + ObjectStoragePtr object_storage, + const DataLakeStorageSettings & data_lake_settings, + const PersistentTableComponents & persistent_components, + const String & write_format, + std::shared_ptr catalog, + const String & table_name, + ExternalStorageCache & external_storages); + +} + +#endif diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/AlterDropPartitionExecutor.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/AlterDropPartitionExecutor.cpp index 54812df2effcf..88786a777e386 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/AlterDropPartitionExecutor.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/AlterDropPartitionExecutor.cpp @@ -92,21 +92,6 @@ static constexpr auto MAX_TRANSACTION_RETRIES = 100; namespace { -/// Find the partition spec object with the given spec-id inside a metadata JSON document. -/// Throws METADATA_MISMATCH if the spec is not found (indicates metadata/spec-id mismatch). -Poco::JSON::Object::Ptr lookupPartitionSpec(const Poco::JSON::Object::Ptr & meta, Int64 spec_id) -{ - auto specs = meta->getArray(f_partition_specs); - for (size_t i = 0; i < specs->size(); ++i) - { - auto spec = specs->getObject(static_cast(i)); - if (spec->getValue(f_spec_id) == spec_id) - return spec; - } - throw Exception(ErrorCodes::METADATA_MISMATCH, - "Partition spec with id {} not found in table metadata", spec_id); -} - /// One term per partition field in spec order PartitionSpecSignature getPartitionSpecSignature(const Poco::JSON::Array::Ptr & spec_fields) { diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp index 24cdfe3e7cbcb..a68d4e8508a4e 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp @@ -160,23 +160,7 @@ static bool isCurrentManifestListAboveThreshold( { LoggerPtr log = getLogger("IcebergCompaction::isCurrentManifestListAboveThreshold"); - if (!metadata_object->has(Iceberg::f_current_snapshot_id)) - return false; - Int64 current_snapshot_id = metadata_object->getValue(Iceberg::f_current_snapshot_id); - if (current_snapshot_id < 0) - return false; - - String current_manifest_list_path; - auto snapshots = metadata_object->get(Iceberg::f_snapshots).extract(); - for (size_t i = 0; i < snapshots->size(); ++i) - { - const auto snapshot = snapshots->getObject(static_cast(i)); - if (snapshot->getValue(Iceberg::f_metadata_snapshot_id) == current_snapshot_id) - { - current_manifest_list_path = snapshot->getValue(Iceberg::f_manifest_list); - break; - } - } + String current_manifest_list_path = getCurrentManifestListPath(metadata_object); if (current_manifest_list_path.empty()) return false; @@ -983,9 +967,7 @@ static bool writeConsolidatedManifestFile( // Commit: write metadata file with If-None-Match + ETag-based CAS version hint; returns false if another writer claimed this version, so the caller retries. { - std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM - Poco::JSON::Stringifier::stringify(metadata_object, oss, 4); - std::string json_representation = removeEscapedSlashes(oss.str()); + std::string json_representation = stringifyJSON(metadata_object, 4); auto hint_path = generator.generateVersionHint(); LOG_INFO(log, "Committing metadata file: {}", diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp index dd2bd1ae96979..9aca0923cdd3f 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp @@ -73,6 +73,7 @@ #include #include #include +#include #include #include #include @@ -167,6 +168,7 @@ extern const SettingsBool allow_experimental_iceberg_compaction; extern const SettingsBool allow_experimental_geo_types_in_iceberg; extern const SettingsBool allow_iceberg_remove_orphan_files; extern const SettingsBool allow_experimental_expire_snapshots; +extern const SettingsBool allow_experimental_iceberg_add_files; extern const SettingsSeconds lock_acquire_timeout; extern const SettingsSeconds iceberg_compaction_delay_bias; extern const SettingsSeconds iceberg_compaction_data_cleanup; @@ -1152,6 +1154,21 @@ Pipe IcebergMetadata::executeCommand( return Iceberg::executeRemoveOrphanFiles( args, context, object_storage_, data_lake_settings, persistent_components, *external_storages); } + else if (command_name == "add_files") + { + if (!context->getSettingsRef()[Setting::allow_experimental_iceberg_add_files].value) + { + throw Exception( + ErrorCodes::SUPPORT_IS_DISABLED, + "Iceberg add_files is experimental. " + "To allow its usage, enable setting allow_experimental_iceberg_add_files"); + } + + checkTableRootIsQueriedPath("add_files"); + return Iceberg::executeAddFiles( + args, context, object_storage_, data_lake_settings, persistent_components, + write_format, catalog_, storage_id.getTableName(), *external_storages); + } else { throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown EXECUTE command '{}' for Iceberg table", command_name); @@ -2266,11 +2283,6 @@ namespace FailPoints extern const char iceberg_writes_post_publish_throw[]; } -namespace -{ - -/// Find the partition spec object with the given spec-id inside a metadata JSON document. -/// Throws METADATA_MISMATCH if the spec is not found (indicates metadata/spec-id mismatch). Poco::JSON::Object::Ptr lookupPartitionSpec(const Poco::JSON::Object::Ptr & meta, Int64 spec_id) { auto specs = meta->getArray(Iceberg::f_partition_specs); @@ -2298,6 +2310,9 @@ Poco::JSON::Object::Ptr lookupSchema(const Poco::JSON::Object::Ptr & meta, Int64 "Schema with id {} not found in table metadata", schema_id); } +namespace +{ + /// Derive the Iceberg partition tuple for an exported part from a representative source row. /// The MergeTree `partition.value` is the source partition-key expression result; it is neither /// cast to the destination column types nor expressed through the Iceberg transform, so it must @@ -2579,9 +2594,7 @@ std::optional IcebergMetadata::commitImport } { - std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM - Poco::JSON::Stringifier::stringify(metadata, oss, 4); - std::string json_representation = removeEscapedSlashes(oss.str()); + std::string json_representation = stringifyJSON(metadata, 4); LOG_DEBUG(log, "Writing new metadata file {}", storage_metadata_name); auto hint_path = filename_generator.generateVersionHint(); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h index f365d5af9f077..d9718b0453b43 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h @@ -350,6 +350,11 @@ class IcebergMetadata : public IDataLakeMetadata std::shared_ptr catalog, const StorageID & storage_id); }; + +/// Find the partition spec / schema object by id inside a metadata JSON document. +/// Throws METADATA_MISMATCH if not found. +Poco::JSON::Object::Ptr lookupPartitionSpec(const Poco::JSON::Object::Ptr & metadata, Int64 spec_id); +Poco::JSON::Object::Ptr lookupSchema(const Poco::JSON::Object::Ptr & metadata, Int64 schema_id); } #endif diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp index b22a389b7ef5f..4fef118fbe08e 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp @@ -1,7 +1,9 @@ #include +#include #include #include +#include #include #include #include @@ -2336,6 +2338,55 @@ PartitionColumnValues getIdentityPartitionColumnValues( return result; } +std::unordered_map collectIcebergTypesByFieldId(Poco::JSON::Array::Ptr schema_fields) +{ + std::unordered_map result; + + std::function collect_types; + collect_types = [&](Poco::JSON::Object::Ptr field, const String & prefix) + { + String name = prefix.empty() ? field->getValue(Iceberg::f_name) : prefix + "." + field->getValue(Iceberg::f_name); + Int64 id = field->getValue(Iceberg::f_id); + auto type_var = field->get(Iceberg::f_type); + if (type_var.isString()) + { + result[id] = type_var.extract(); + } + else if (type_var.isStruct()) + { + auto type_obj = type_var.extract(); + String kind = type_obj->getValue(Iceberg::f_type); + if (kind == "struct") + { + for (const auto & nested : *type_obj->getArray(Iceberg::f_fields)) + collect_types(nested.extract(), name); + } + } + }; + for (const auto & field_var : *schema_fields) + collect_types(field_var.extract(), ""); + + return result; +} + +String getCurrentManifestListPath(const Poco::JSON::Object::Ptr & metadata) +{ + if (!metadata->has(Iceberg::f_current_snapshot_id) || metadata->isNull(Iceberg::f_current_snapshot_id)) + return {}; + + Int64 current_snapshot_id = metadata->getValue(Iceberg::f_current_snapshot_id); + if (current_snapshot_id < 0) + return {}; + + for (const auto & snapshot_var : *metadata->getArray(Iceberg::f_snapshots)) + { + auto snapshot = snapshot_var.extract(); + if (snapshot->getValue(Iceberg::f_metadata_snapshot_id) == current_snapshot_id) + return snapshot->getValue(Iceberg::f_manifest_list); + } + return {}; +} + } #endif diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h index c06cfe0fb8340..397c47bb1b59c 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h @@ -227,6 +227,13 @@ using PartitionColumnValues = std::vector>; PartitionColumnValues getIdentityPartitionColumnValues( const ProcessedManifestFileEntry & manifest_file_entry, const IcebergSchemaProcessor & schema_processor); + +/// Walk a schema's fields array and return field-id -> raw Iceberg type string +/// (e.g. "date", "timestamptz") for every leaf field, including nested struct members. +std::unordered_map collectIcebergTypesByFieldId(Poco::JSON::Array::Ptr schema_fields); + +/// Returns the manifest-list path of the current snapshot, or empty string if no current snapshot. +String getCurrentManifestListPath(const Poco::JSON::Object::Ptr & metadata); } #endif diff --git a/tests/integration/test_storage_iceberg_with_spark/test_add_files.py b/tests/integration/test_storage_iceberg_with_spark/test_add_files.py new file mode 100644 index 0000000000000..f48fce9012327 --- /dev/null +++ b/tests/integration/test_storage_iceberg_with_spark/test_add_files.py @@ -0,0 +1,622 @@ +import io +import json +import os +import re +import tempfile +from decimal import Decimal + +import pyarrow as pa +import pyarrow.parquet as pq +import pytest + +from helpers.iceberg_utils import ( + create_iceberg_table, + default_download_directory, + get_uuid_str, +) + + +ICEBERG_SETTINGS = { + "allow_insert_into_iceberg": 1, + "allow_experimental_iceberg_add_files": 1, +} + +# Path of the table location in metadata form (with a leading slash); this is the +# form `add_files` expects, the same one native writes register in manifests. +TABLE_PREFIX = "/var/lib/clickhouse/user_files/iceberg_data/default" +# Storage key prefix inside the MinIO bucket (no leading slash). +S3_KEY_PREFIX = "var/lib/clickhouse/user_files/iceberg_data/default" + + +# --------------------------------------------------------------------------- +# Test environment — binds cluster/instance/storage_type/table_name once +# so individual tests only express the add_files-specific logic. +# --------------------------------------------------------------------------- + +class AddFilesTestEnv: + def __init__(self, cluster, storage_type, table_prefix, schema="(x Int32)", format_version=2, partition_by=""): + self.cluster = cluster + self.instance = cluster.instances["node1"] + self.storage_type = storage_type + self.table_name = f"{table_prefix}_{storage_type}_{get_uuid_str()}" + create_iceberg_table( + storage_type, self.instance, self.table_name, + self.cluster, schema, format_version, + partition_by=partition_by, + ) + + # -- paths --------------------------------------------------------------- + + def table_path(self, subdir, filename): + """Metadata-form path of a file below the table root, as passed to add_files.""" + return f"{TABLE_PREFIX}/{self.table_name}/{subdir}/{filename}" + + # -- standalone parquet files --------------------------------------------- + + def write_parquet(self, subdir, filename, select_sql): + """Produce a standalone parquet file (written by ClickHouse itself, so the + footer matches what Iceberg writers produce) at {table}/{subdir}/{filename}.""" + if self.storage_type == "local": + target_dir = f"{TABLE_PREFIX}/{self.table_name}/{subdir}" + self.instance.exec_in_container(["bash", "-c", f"mkdir -p {target_dir}"]) + self.instance.query( + f"INSERT INTO TABLE FUNCTION " + f"file('iceberg_data/default/{self.table_name}/{subdir}/{filename}') " + f"{select_sql} FORMAT Parquet" + ) + else: + # Write the file inside the container and pull it out with a + # binary-safe copy; the host instance dir is not assumed to be + # reachable from the pytest side. + tmp_filename = f"add_files_tmp_{get_uuid_str()}.parquet" + container_path = f"/var/lib/clickhouse/user_files/{tmp_filename}" + self.instance.query( + f"INSERT INTO TABLE FUNCTION " + f"file('{tmp_filename}') " + f"{select_sql} FORMAT Parquet" + ) + host_path = os.path.join(tempfile.gettempdir(), tmp_filename) + self.instance.copy_file_from_container(container_path, host_path) + with open(host_path, "rb") as f: + data = f.read() + os.remove(host_path) + self.instance.exec_in_container(["rm", "-f", container_path]) + self.put_file(subdir, filename, data) + + def write_parquet_arrow(self, subdir, filename, table, **write_options): + """Write a parquet file with pyarrow, to get representations that ClickHouse itself + does not produce (other timestamp units, missing statistics, ...).""" + buffer = io.BytesIO() + pq.write_table(table, buffer, **write_options) + self.put_file(subdir, filename, buffer.getvalue()) + + def put_file(self, subdir, filename, data): + if self.storage_type == "local": + host_path = os.path.join(tempfile.gettempdir(), f"add_files_tmp_{get_uuid_str()}.parquet") + with open(host_path, "wb") as f: + f.write(data) + target_dir = f"{TABLE_PREFIX}/{self.table_name}/{subdir}" + self.instance.exec_in_container(["bash", "-c", f"mkdir -p {target_dir}"]) + self.instance.copy_file_to_container(host_path, f"{target_dir}/{filename}") + os.remove(host_path) + else: + key = f"{S3_KEY_PREFIX}/{self.table_name}/{subdir}/{filename}" + self.cluster.minio_client.put_object( + self.cluster.minio_bucket, key, io.BytesIO(data), len(data), + ) + + # -- command execution ----------------------------------------------------- + + def add_files_query(self, paths, check_duplicate_files=None, allow_missing_columns=None): + if isinstance(paths, str): + paths_sql = f"'{paths}'" + else: + paths_sql = "[" + ", ".join(f"'{p}'" for p in paths) + "]" + args = [paths_sql] + if check_duplicate_files is not None: + args.append(f"check_duplicate_files = {check_duplicate_files}") + if allow_missing_columns is not None: + args.append(f"allow_missing_columns = {allow_missing_columns}") + return f"ALTER TABLE {self.table_name} EXECUTE add_files({', '.join(args)});" + + @staticmethod + def parse_metrics(raw): + counts = {} + for line in raw.strip().split("\n"): + if not line: + continue + parts = line.split("\t") + if len(parts) == 2: + counts[parts[0]] = int(parts[1]) + return counts + + def add_files(self, paths, **kwargs): + raw = self.instance.query(self.add_files_query(paths, **kwargs), settings=ICEBERG_SETTINGS) + return self.parse_metrics(raw) + + def add_files_error(self, paths, **kwargs): + return self.instance.query_and_get_error(self.add_files_query(paths, **kwargs), settings=ICEBERG_SETTINGS) + + # -- helpers ---------------------------------------------------------------- + + def insert(self, values): + self.instance.query(f"INSERT INTO {self.table_name} VALUES {values};", settings=ICEBERG_SETTINGS) + + def query(self, sql): + return self.instance.query(sql, settings=ICEBERG_SETTINGS) + + def count(self): + return int(self.query(f"SELECT count() FROM {self.table_name}")) + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +@pytest.mark.parametrize("format_version", [1, 2]) +def test_add_files_basic(started_cluster_iceberg_with_spark, storage_type, format_version): + """Register one external parquet file into an empty table and read it back.""" + env = AddFilesTestEnv(started_cluster_iceberg_with_spark, storage_type, "test_add_basic", format_version=format_version) + + env.write_parquet("data/import", "f1.parquet", "SELECT toInt32(number + 1) AS x FROM numbers(3)") + counts = env.add_files(env.table_path("data/import", "f1.parquet")) + + assert counts["files_registered"] == 1, f"got {counts}" + assert counts["rows_registered"] == 3, f"got {counts}" + assert counts["manifests_written"] == 1, f"got {counts}" + assert counts["bytes_registered"] > 0, f"got {counts}" + assert env.query(f"SELECT * FROM {env.table_name} ORDER BY x") == "1\n2\n3\n" + + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +def test_add_files_array_form_multiple(started_cluster_iceberg_with_spark, storage_type): + """The array form registers several files in one snapshot.""" + env = AddFilesTestEnv(started_cluster_iceberg_with_spark, storage_type, "test_add_array") + + env.write_parquet("data/import", "f1.parquet", "SELECT toInt32(number) AS x FROM numbers(3)") + env.write_parquet("data/import", "f2.parquet", "SELECT toInt32(number + 10) AS x FROM numbers(3)") + counts = env.add_files([ + env.table_path("data/import", "f1.parquet"), + env.table_path("data/import", "f2.parquet"), + ]) + + assert counts["files_registered"] == 2, f"got {counts}" + assert counts["rows_registered"] == 6, f"got {counts}" + assert env.count() == 6 + assert env.query(f"SELECT * FROM {env.table_name} ORDER BY x") == "0\n1\n2\n10\n11\n12\n" + + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +def test_add_files_onto_nonempty_table(started_cluster_iceberg_with_spark, storage_type): + """Registration appends a snapshot on top of natively-written data.""" + env = AddFilesTestEnv(started_cluster_iceberg_with_spark, storage_type, "test_add_nonempty") + + env.insert("(1), (2)") + env.write_parquet("data/import", "f1.parquet", "SELECT toInt32(number + 100) AS x FROM numbers(3)") + counts = env.add_files(env.table_path("data/import", "f1.parquet")) + + assert counts["files_registered"] == 1, f"got {counts}" + assert env.count() == 5 + assert env.query(f"SELECT * FROM {env.table_name} ORDER BY x") == "1\n2\n100\n101\n102\n" + + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +def test_add_files_trailing_slash_expansion(started_cluster_iceberg_with_spark, storage_type): + """A path ending with '/' registers every *.parquet below it, skipping hidden files.""" + env = AddFilesTestEnv(started_cluster_iceberg_with_spark, storage_type, "test_add_expand") + + env.write_parquet("data/import", "f1.parquet", "SELECT toInt32(1) AS x") + env.write_parquet("data/import", "f2.parquet", "SELECT toInt32(2) AS x") + env.write_parquet("data/import", ".hidden.parquet", "SELECT toInt32(3) AS x") + + counts = env.add_files(env.table_path("data/import", "")) + + assert counts["files_registered"] == 2, f"got {counts}" + assert env.count() == 2 + assert env.query(f"SELECT * FROM {env.table_name} ORDER BY x") == "1\n2\n" + + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +def test_add_files_partitioned_identity(started_cluster_iceberg_with_spark, storage_type): + """Single-partition files register into an identity-partitioned table; a file + spanning two partition values is rejected.""" + env = AddFilesTestEnv( + started_cluster_iceberg_with_spark, storage_type, "test_add_identity", + partition_by="identity(x)", + ) + + env.write_parquet("data/import", "f7.parquet", "SELECT toInt32(7) AS x FROM numbers(3)") + counts = env.add_files(env.table_path("data/import", "f7.parquet")) + assert counts["files_registered"] == 1, f"got {counts}" + assert counts["partitions_touched"] == 1, f"got {counts}" + + env.write_parquet("data/import", "f8.parquet", "SELECT toInt32(8) AS x FROM numbers(2)") + counts = env.add_files(env.table_path("data/import", "f8.parquet")) + assert counts["partitions_touched"] == 1, f"got {counts}" + assert env.query(f"SELECT * FROM {env.table_name} ORDER BY x") == "7\n7\n7\n8\n8\n" + + env.write_parquet("data/import", "mixed.parquet", "SELECT toInt32(7) AS x UNION ALL SELECT toInt32(8) AS x") + error = env.add_files_error(env.table_path("data/import", "mixed.parquet")) + assert "spans more than one partition" in error, f"got {error}" + # The failed registration must not have committed anything. + assert env.count() == 5 + + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +def test_add_files_day_transform(started_cluster_iceberg_with_spark, storage_type): + """The day() transform is validated from footer bounds: same-day file registers, + a file crossing midnight is rejected.""" + env = AddFilesTestEnv( + started_cluster_iceberg_with_spark, storage_type, "test_add_day", + schema="(d DateTime64(6))", + partition_by="toRelativeDayNum(d)", + ) + + env.write_parquet( + "data/import", "same_day.parquet", + "SELECT toDateTime64('2024-01-01 10:00:00', 6) AS d " + "UNION ALL SELECT toDateTime64('2024-01-01 23:00:00', 6) AS d", + ) + counts = env.add_files(env.table_path("data/import", "same_day.parquet")) + assert counts["files_registered"] == 1, f"got {counts}" + assert env.count() == 2 + + env.write_parquet( + "data/import", "two_days.parquet", + "SELECT toDateTime64('2024-01-01 10:00:00', 6) AS d " + "UNION ALL SELECT toDateTime64('2024-01-02 10:00:00', 6) AS d", + ) + error = env.add_files_error(env.table_path("data/import", "two_days.parquet")) + assert "spans more than one partition" in error, f"got {error}" + assert env.count() == 2 + + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +def test_add_files_duplicate_detection(started_cluster_iceberg_with_spark, storage_type): + """Duplicates against the current snapshot and inside the input are rejected by + default; check_duplicate_files = 0 disables the check.""" + env = AddFilesTestEnv(started_cluster_iceberg_with_spark, storage_type, "test_add_dup") + + env.write_parquet("data/import", "f1.parquet", "SELECT toInt32(1) AS x") + path = env.table_path("data/import", "f1.parquet") + + counts = env.add_files(path) + assert counts["files_registered"] == 1, f"got {counts}" + + error = env.add_files_error(path) + assert "already registered in the current snapshot" in error, f"got {error}" + + error = env.add_files_error([path, path]) + assert "listed twice" in error, f"got {error}" + + counts = env.add_files(path, check_duplicate_files=0) + assert counts["files_registered"] == 1, f"got {counts}" + assert env.count() == 2 + + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +def test_add_files_allow_missing_columns(started_cluster_iceberg_with_spark, storage_type): + """A file lacking a schema column is rejected by default and readable with + allow_missing_columns = 1 (missing values read as NULL).""" + env = AddFilesTestEnv( + started_cluster_iceberg_with_spark, storage_type, "test_add_missing", + schema="(x Int32, y Nullable(String))", + ) + + env.write_parquet("data/import", "only_x.parquet", "SELECT toInt32(1) AS x") + path = env.table_path("data/import", "only_x.parquet") + + error = env.add_files_error(path) + assert "lacks column" in error, f"got {error}" + assert "'y'" in error, f"got {error}" + + counts = env.add_files(path, allow_missing_columns=1) + assert counts["files_registered"] == 1, f"got {counts}" + assert env.query(f"SELECT x, y FROM {env.table_name}") == "1\t\\N\n" + + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +def test_add_files_argument_errors(started_cluster_iceberg_with_spark, storage_type): + """Missing files, non-parquet paths, and bucket-partitioned tables are rejected.""" + env = AddFilesTestEnv(started_cluster_iceberg_with_spark, storage_type, "test_add_args") + + error = env.add_files_error(env.table_path("data", "nonexistent.parquet")) + assert "does not exist" in error, f"got {error}" + + error = env.add_files_error(env.table_path("data", "f1.orc")) + assert ".parquet" in error, f"got {error}" + + env_bucket = AddFilesTestEnv( + started_cluster_iceberg_with_spark, storage_type, "test_add_bucket", + partition_by="icebergBucket(4, x)", + ) + env_bucket.write_parquet("data/import", "f1.parquet", "SELECT toInt32(1) AS x") + error = env_bucket.add_files_error(env_bucket.table_path("data/import", "f1.parquet")) + assert "bucket transform" in error, f"got {error}" + + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +def test_add_files_rejects_mismatched_types(started_cluster_iceberg_with_spark, storage_type): + """Footer statistics are used as Iceberg bounds as they are, so a file whose column is stored + differently from the Iceberg type must be rejected. A matching file registers, and filters + on its bounds return every matching row.""" + env = AddFilesTestEnv( + started_cluster_iceberg_with_spark, storage_type, "test_add_types", + schema="(x Nullable(Int64), d Nullable(DateTime64(6)))", + ) + + timestamps = [1704103200000000 + i * 1000000 for i in range(10)] # 2024-01-01 10:00:00 + i seconds + + # INT32 data for an Iceberg `long` column: the bounds would be 4 bytes instead of 8. + env.write_parquet_arrow("data/import", "int32.parquet", pa.table({ + "x": pa.array(range(1, 11), type=pa.int32()), + "d": pa.array(timestamps, type=pa.timestamp("us")), + })) + error = env.add_files_error(env.table_path("data/import", "int32.parquet")) + assert "does not match the Iceberg type 'long'" in error, f"got {error}" + + # Nanosecond timestamps: the bounds would be read as microseconds. + env.write_parquet_arrow("data/import", "ns.parquet", pa.table({ + "x": pa.array(range(1, 11), type=pa.int64()), + "d": pa.array([t * 1000 for t in timestamps], type=pa.timestamp("ns")), + })) + error = env.add_files_error(env.table_path("data/import", "ns.parquet")) + assert "does not match the Iceberg type" in error, f"got {error}" + + # A string column for an integer field. + env.write_parquet_arrow("data/import", "string.parquet", pa.table({ + "x": pa.array([str(i) for i in range(1, 11)], type=pa.string()), + "d": pa.array(timestamps, type=pa.timestamp("us")), + })) + error = env.add_files_error(env.table_path("data/import", "string.parquet")) + assert "does not match the Iceberg type 'long'" in error, f"got {error}" + + assert env.count() == 0 + + env.write_parquet_arrow("data/import", "ok.parquet", pa.table({ + "x": pa.array(range(1, 11), type=pa.int64()), + "d": pa.array(timestamps, type=pa.timestamp("us")), + })) + counts = env.add_files(env.table_path("data/import", "ok.parquet")) + assert counts["files_registered"] == 1, f"got {counts}" + assert env.query(f"SELECT count() FROM {env.table_name} WHERE x = 5") == "1\n" + assert env.query(f"SELECT count() FROM {env.table_name} WHERE x > 8") == "2\n" + assert env.query( + f"SELECT count() FROM {env.table_name} WHERE d >= toDateTime64('2024-01-01 10:00:08', 6, 'UTC')" + ) == "2\n" + + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +def test_add_files_partition_value_needs_statistics(started_cluster_iceberg_with_spark, storage_type): + """The partition value is derived from footer statistics. A file without them must be rejected + instead of being registered in the null partition; a file whose values are all null does belong + to the null partition.""" + env = AddFilesTestEnv( + started_cluster_iceberg_with_spark, storage_type, "test_add_no_stats", + schema="(s Nullable(String))", + partition_by="identity(s)", + ) + + values = pa.table({"s": pa.array(["abc"] * 3, type=pa.string())}) + env.write_parquet_arrow("data/import", "no_stats.parquet", values, write_statistics=False) + error = env.add_files_error(env.table_path("data/import", "no_stats.parquet")) + assert "no complete min/max statistics" in error, f"got {error}" + assert env.count() == 0 + + env.write_parquet_arrow("data/import", "stats.parquet", values) + counts = env.add_files(env.table_path("data/import", "stats.parquet")) + assert counts["partitions_touched"] == 1, f"got {counts}" + + env.write_parquet_arrow("data/import", "nulls.parquet", pa.table({"s": pa.array([None, None], type=pa.string())})) + counts = env.add_files(env.table_path("data/import", "nulls.parquet")) + assert counts["partitions_touched"] == 1, f"got {counts}" + + assert env.query(f"SELECT count() FROM {env.table_name} WHERE s = 'abc'") == "3\n" + assert env.query(f"SELECT count() FROM {env.table_name} WHERE s IS NULL") == "2\n" + assert env.count() == 5 + + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +def test_add_files_requires_setting(started_cluster_iceberg_with_spark, storage_type): + """add_files is experimental and gated by its own setting.""" + env = AddFilesTestEnv(started_cluster_iceberg_with_spark, storage_type, "test_add_setting") + env.write_parquet("data/import", "f1.parquet", "SELECT toInt32(1) AS x") + + error = env.instance.query_and_get_error( + env.add_files_query(env.table_path("data/import", "f1.parquet")), + settings={"allow_insert_into_iceberg": 1}, + ) + assert "allow_experimental_iceberg_add_files" in error, f"got {error}" + assert env.count() == 0 + + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +def test_add_files_rejects_paths_outside_table_location(started_cluster_iceberg_with_spark, storage_type): + """Files are read and registered through the table's storage, so only files under the table + location may be registered: not the files of another table, not a sibling directory sharing + the prefix, and not a path escaping the location with '..'.""" + env = AddFilesTestEnv(started_cluster_iceberg_with_spark, storage_type, "test_add_outside") + other = AddFilesTestEnv(started_cluster_iceberg_with_spark, storage_type, "test_add_outside_other") + other.write_parquet("data/import", "f1.parquet", "SELECT toInt32(1) AS x") + + for path in [ + other.table_path("data/import", "f1.parquet"), + other.table_path("data/import", ""), + f"{TABLE_PREFIX}/{env.table_name}_sibling/data/f1.parquet", + ]: + error = env.add_files_error(path) + assert "is not under the table location" in error, f"{path}: got {error}" + + error = env.add_files_error(env.table_path("data", f"../../{other.table_name}/data/import/f1.parquet")) + assert "relative components" in error, f"got {error}" + + assert env.count() == 0 + + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +def test_add_files_empty_directory(started_cluster_iceberg_with_spark, storage_type): + """A directory without parquet files is an error instead of an empty snapshot.""" + env = AddFilesTestEnv(started_cluster_iceberg_with_spark, storage_type, "test_add_empty_dir") + env.write_parquet("data/import", ".hidden.parquet", "SELECT toInt32(1) AS x") + + error = env.add_files_error(env.table_path("data/import", "")) + assert "contains no parquet files" in error, f"got {error}" + + snapshots = env.query(f"SELECT count() FROM system.iceberg_history WHERE table = '{env.table_name}'") + assert snapshots == "0\n", f"got {snapshots}" + + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +def test_add_files_missing_required_column(started_cluster_iceberg_with_spark, storage_type): + """A missing column reads as NULL, so allow_missing_columns = 1 does not cover required fields.""" + env = AddFilesTestEnv( + started_cluster_iceberg_with_spark, storage_type, "test_add_missing_required", + schema="(x Int32, y Int32)", + ) + env.write_parquet_arrow("data/import", "only_x.parquet", pa.table({"x": pa.array([1], type=pa.int32())})) + path = env.table_path("data/import", "only_x.parquet") + + error = env.add_files_error(path, allow_missing_columns=1) + assert "required in the table schema" in error, f"got {error}" + assert "'y'" in error, f"got {error}" + assert env.count() == 0 + + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +def test_add_files_rejects_unsupported_representations(started_cluster_iceberg_with_spark, storage_type): + """Decimal columns and millisecond timestamps would need their bounds converted to the Iceberg + encoding, so they are rejected rather than registered with wrong bounds.""" + env_decimal = AddFilesTestEnv( + started_cluster_iceberg_with_spark, storage_type, "test_add_decimal", + schema="(x Nullable(Decimal(9, 2)))", + ) + env_decimal.write_parquet_arrow("data/import", "decimal.parquet", pa.table({ + "x": pa.array([Decimal("1.00"), Decimal("-2.00")], type=pa.decimal128(9, 2)), + })) + error = env_decimal.add_files_error(env_decimal.table_path("data/import", "decimal.parquet")) + assert "which add_files does not support yet" in error, f"got {error}" + assert env_decimal.count() == 0 + + env_ts = AddFilesTestEnv( + started_cluster_iceberg_with_spark, storage_type, "test_add_ts_millis", + schema="(d Nullable(DateTime64(6)))", + partition_by="toRelativeDayNum(d)", + ) + env_ts.write_parquet_arrow("data/import", "millis.parquet", pa.table({ + "d": pa.array([1704103200000, 1704103201000], type=pa.timestamp("ms")), + }), coerce_timestamps="ms") + error = env_ts.add_files_error(env_ts.table_path("data/import", "millis.parquet")) + assert "does not match the Iceberg type" in error, f"got {error}" + assert env_ts.count() == 0 + + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +def test_add_files_truncate_multibyte(started_cluster_iceberg_with_spark, storage_type): + """truncate[N] keeps N code points, as a native insert does: a registered file lands in the same + partition as natively inserted rows, so partition pruning keeps both.""" + env = AddFilesTestEnv( + started_cluster_iceberg_with_spark, storage_type, "test_add_truncate_utf8", + schema="(s Nullable(String))", + partition_by="icebergTruncate(2, s)", + ) + env.insert("('äöü')") + + env.write_parquet_arrow("data/import", "same.parquet", pa.table({"s": pa.array(["äöü", "äöx"], type=pa.string())})) + counts = env.add_files(env.table_path("data/import", "same.parquet")) + assert counts["partitions_touched"] == 1, f"got {counts}" + + # 'äb' and 'äc' share their first 2 bytes, but not their first 2 code points. + env.write_parquet_arrow("data/import", "two.parquet", pa.table({"s": pa.array(["äb", "äc"], type=pa.string())})) + error = env.add_files_error(env.table_path("data/import", "two.parquet")) + assert "spans more than one partition" in error, f"got {error}" + + pruning = {"use_iceberg_partition_pruning": 1} + assert env.instance.query( + f"SELECT count() FROM {env.table_name} WHERE s = 'äöü'", settings=pruning + ) == "2\n" + assert env.instance.query( + f"SELECT count() FROM {env.table_name} WHERE s = 'äöx'", settings=pruning + ) == "1\n" + assert env.count() == 3 + + +def download_for_spark(cluster, storage_type, table_name): + """Copy the table to the host for Spark and point the version hint at the latest metadata file. + Returns the local table directory and the parsed latest metadata.""" + local_dir = f"/var/lib/clickhouse/user_files/iceberg_data/default/{table_name}/" + default_download_directory(cluster, storage_type, local_dir, local_dir) + metadata_dir = os.path.join(local_dir, "metadata") + versions = {} + for name in os.listdir(metadata_dir): + match = re.match(r"v(\d+)(?:[-.].*)?\.metadata\.json$", name) + if match: + versions[int(match.group(1))] = name + latest = max(versions) + with open(os.path.join(metadata_dir, "version-hint.text"), "w") as hint: + hint.write(str(latest)) + with open(os.path.join(metadata_dir, versions[latest])) as f: + return local_dir, json.load(f) + + +def name_mapping_of(metadata): + """Field id -> names of the default name mapping, over all nesting levels.""" + result = {} + + def walk(entries): + for entry in entries: + result[entry["field-id"]] = entry["names"] + walk(entry.get("fields", [])) + + walk(json.loads(metadata["properties"]["schema.name-mapping.default"])) + return result + + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +def test_add_files_spark_reads_by_name(started_cluster_iceberg_with_spark, storage_type): + """Registered files carry no Iceberg field ids. add_files sets the default name mapping, so Spark + maps their columns by name, even when the file's column order differs from the schema.""" + cluster = started_cluster_iceberg_with_spark + env = AddFilesTestEnv( + cluster, storage_type, "test_add_spark", + schema="(a Nullable(Int32), b Nullable(String))", + ) + env.write_parquet_arrow("data/import", "reordered.parquet", pa.table({ + "b": pa.array(["x", "y"], type=pa.string()), + "a": pa.array([1, 2], type=pa.int32()), + })) + env.add_files(env.table_path("data/import", "reordered.parquet")) + assert env.query(f"SELECT a, b FROM {env.table_name} ORDER BY a") == "1\tx\n2\ty\n" + + local_dir, metadata = download_for_spark(cluster, storage_type, env.table_name) + assert sorted(names[0] for names in name_mapping_of(metadata).values()) == ["a", "b"] + rows = cluster.spark_session.read.format("iceberg").load(local_dir).collect() + assert sorted((row["a"], row["b"]) for row in rows) == [(1, "x"), (2, "y")] + + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +def test_add_files_nulls_in_required_column(started_cluster_iceberg_with_spark, storage_type): + """A required field must have a value in every row, so a file with nulls in its column, or without + the null counts that prove it has none, is rejected. A column that cannot hold nulls needs no counts.""" + env = AddFilesTestEnv( + started_cluster_iceberg_with_spark, storage_type, "test_add_required_nulls", + schema="(x Int32)", + ) + + env.write_parquet_arrow("data/import", "nulls.parquet", pa.table({"x": pa.array([1, None], type=pa.int32())})) + error = env.add_files_error(env.table_path("data/import", "nulls.parquet")) + assert "contains 1 null(s)" in error, f"got {error}" + + env.write_parquet_arrow( + "data/import", "no_stats.parquet", pa.table({"x": pa.array([1, 2], type=pa.int32())}), write_statistics=False) + error = env.add_files_error(env.table_path("data/import", "no_stats.parquet")) + assert "may contain nulls" in error, f"got {error}" + assert env.count() == 0 + + required = pa.schema([pa.field("x", pa.int32(), nullable=False)]) + env.write_parquet_arrow( + "data/import", "required.parquet", pa.table({"x": pa.array([1, 2], type=pa.int32())}, schema=required), + write_statistics=False) + counts = env.add_files(env.table_path("data/import", "required.parquet")) + assert counts["files_registered"] == 1, f"got {counts}" + assert env.count() == 2