From b3fe55f134f40767874972066ff91f587f9dc820 Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Sun, 1 Nov 2026 15:34:34 +0100 Subject: [PATCH 1/9] Added feature to add files in iceberg --- .../DataLakes/Iceberg/AddFilesExecute.cpp | 1139 +++++++++++++++++ .../DataLakes/Iceberg/AddFilesExecute.h | 58 + .../DataLakes/Iceberg/IcebergMetadata.cpp | 8 + .../test_add_files.py | 310 +++++ 4 files changed, 1515 insertions(+) create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/AddFilesExecute.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/AddFilesExecute.h create mode 100644 tests/integration/test_storage_iceberg_with_spark/test_add_files.py diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/AddFilesExecute.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/AddFilesExecute.cpp new file mode 100644 index 0000000000000..d2d735e738621 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/AddFilesExecute.cpp @@ -0,0 +1,1139 @@ +#include + +#include "config.h" + +#if USE_AVRO + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ +extern const int BAD_ARGUMENTS; +extern const int ICEBERG_SPECIFICATION_VIOLATION; +extern const int LOGICAL_ERROR; +extern const int METADATA_MISMATCH; +extern const int SUPPORT_IS_DISABLED; +} + +namespace DataLakeStorageSetting +{ +extern const DataLakeStorageSettingsBool iceberg_use_version_hint; +} + +namespace Iceberg +{ + +namespace +{ + +ExecuteCommandArgs makeSchema() +{ + ExecuteCommandArgs schema("add_files"); + schema.addPositional("paths"); + schema.addNamed("check_duplicate_files", Field::Types::UInt64); + schema.addNamed("allow_missing_columns", Field::Types::UInt64); + schema.addDefault("check_duplicate_files", Field(UInt64(1))); + schema.addDefault("allow_missing_columns", Field(UInt64(0))); + return schema; +} + +std::vector parsePathsArgument(const Field & field) +{ + std::vector paths; + if (field.getType() == Field::Types::String) + { + paths.push_back(field.safeGet()); + } + else if (field.getType() == Field::Types::Array) + { + for (const auto & item : field.safeGet()) + paths.push_back(item.safeGet()); + } + else + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "add_files expects 'paths' to be a String or an Array of Strings, got {}", field.getTypeName()); + } + if (paths.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "add_files expects at least one path"); + return paths; +} + +bool hasHiddenPathComponent(const String & path) +{ + size_t start = 0; + while (start <= path.size()) + { + size_t slash = path.find('/', start); + String component = path.substr(start, slash == String::npos ? String::npos : slash - start); + if (!component.empty() && component[0] == '.') + return true; + if (slash == String::npos) + break; + start = slash + 1; + } + return false; +} + +bool endsWithParquet(const String & path) +{ + return Poco::toLower(path).ends_with(".parquet"); +} + +/// Mirrors the anonymous-namespace helper in IcebergMetadata.cpp: find the partition spec +/// object with the given spec-id inside a metadata JSON document; throws if absent. +Poco::JSON::Object::Ptr lookupPartitionSpec(const Poco::JSON::Object::Ptr & metadata, Int64 spec_id) +{ + auto specs = metadata->getArray(Iceberg::f_partition_specs); + for (size_t i = 0; i < specs->size(); ++i) + { + auto spec = specs->getObject(static_cast(i)); + if (spec->getValue(Iceberg::f_spec_id) == spec_id) + return spec; + } + throw Exception(ErrorCodes::METADATA_MISMATCH, + "Partition spec with id {} not found in table metadata", spec_id); +} + +/// Find the schema object with the given schema-id inside a metadata JSON document; throws if absent. +Poco::JSON::Object::Ptr lookupSchema(const Poco::JSON::Object::Ptr & metadata, Int64 schema_id) +{ + auto schemas = metadata->getArray(Iceberg::f_schemas); + for (size_t i = 0; i < schemas->size(); ++i) + { + auto schema = schemas->getObject(static_cast(i)); + if (schema->getValue(Iceberg::f_schema_id) == schema_id) + return schema; + } + throw Exception(ErrorCodes::METADATA_MISMATCH, + "Schema with id {} not found in table metadata", schema_id); +} + +/// Expand trailing-slash paths into concrete *.parquet files below them. +std::vector expandPaths( + const std::vector & raw_paths, + const IcebergPathResolver & resolver, + const ObjectStoragePtr & object_storage, + LoggerPtr log) +{ + std::vector result; + for (const auto & raw : raw_paths) + { + if (!raw.ends_with('/')) + { + result.push_back(raw); + continue; + } + RelativePathsWithMetadata children; + object_storage->listObjects(resolver.resolve(IcebergPathFromMetadata::deserialize(raw)), children, /* max_keys */ 0); + for (const auto & child : children) + { + if (!endsWithParquet(child->relative_path) || hasHiddenPathComponent(child->relative_path)) + continue; + /// Listed keys are storage paths; convert to the metadata (location-prefixed) form + /// that native writes register in manifests. + result.push_back(resolver.reverseResolve(child->relative_path).serialize()); + } + LOG_DEBUG(log, "add_files: path {} expanded to {} parquet file(s)", raw, result.size()); + } + return result; +} + +Int64 floorDiv(Int64 a, Int64 b) +{ + Int64 q = a / b; + if ((a % b != 0) && ((a < 0) != (b < 0))) + --q; + return q; +} + +/// Howard Hinnant's civil-from-days algorithm. +void civilFromDays(Int64 z, Int64 & y, unsigned & m, unsigned & d) +{ + z += 719468; + Int64 era = floorDiv(z, 146097); + unsigned doe = static_cast(z - era * 146097); + unsigned yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; + Int64 year = static_cast(yoe) + era * 400; + unsigned doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + unsigned mp = (5 * doy + 2) / 153; + d = doy - (153 * mp + 2) / 5 + 1; + m = mp + (mp < 10 ? 3 : static_cast(-9)); + y = year + (m <= 2 ? 1 : 0); +} + +enum class SourceValueKind +{ + Integer, /// int32 / int64 (also date days and timestamp micros) + String, + Unsupported +}; + +/// Decoded min/max of one parquet column across all row groups, in encoded-bytes form +/// for equality and in decoded form for transforms. +struct ColumnBounds +{ + bool has_bounds = false; + /// For Integer bounds: whether the parquet physical type is INT32 (4-byte LE serialization) + /// rather than INT64 (8-byte). Decided by the physical type, never by the value range. + bool int_is_32bit = false; + Int64 min_int = 0; + Int64 max_int = 0; + String min_str; + String max_str; + SourceValueKind kind = SourceValueKind::Unsupported; +}; + +template +T decodeLE(const String & bytes) +{ + T value = 0; + memcpy(&value, bytes.data(), std::min(bytes.size(), sizeof(T))); + return value; +} + +/// Statistics of one data file derived from its parquet footer. +struct AnalyzedFile +{ + String registration_path; /// path form stored in the manifest + String storage_path; /// resolved path used for reads + IcebergSerializedFileStats stats; + std::unordered_map bounds_by_field_id; +}; + +ColumnBounds aggregateColumnBounds( + const parquet::ColumnChunkMetaData & column, + ColumnBounds acc) +{ + if (!column.is_stats_set()) + return acc; + auto stats = column.statistics(); + if (!stats || !stats->HasMinMax()) + return acc; + + String encoded_min = stats->EncodeMin(); + String encoded_max = stats->EncodeMax(); + auto physical_type = stats->physical_type(); + + SourceValueKind kind = SourceValueKind::Unsupported; + if (physical_type == parquet::Type::INT32 || physical_type == parquet::Type::INT64) + kind = SourceValueKind::Integer; + else if (physical_type == parquet::Type::BYTE_ARRAY || physical_type == parquet::Type::FIXED_LEN_BYTE_ARRAY) + kind = SourceValueKind::String; + + if (kind == SourceValueKind::Unsupported) + return acc; + + if (!acc.has_bounds) + { + acc.kind = kind; + if (kind == SourceValueKind::Integer) + { + acc.int_is_32bit = physical_type == parquet::Type::INT32; + acc.min_int = physical_type == parquet::Type::INT32 ? Int64(decodeLE(encoded_min)) : decodeLE(encoded_min); + acc.max_int = physical_type == parquet::Type::INT32 ? Int64(decodeLE(encoded_max)) : decodeLE(encoded_max); + } + else + { + acc.min_str = encoded_min; + acc.max_str = encoded_max; + } + acc.has_bounds = true; + return acc; + } + + if (kind != acc.kind) + return acc; + + if (kind == SourceValueKind::Integer) + { + Int64 this_min = physical_type == parquet::Type::INT32 ? Int64(decodeLE(encoded_min)) : decodeLE(encoded_min); + Int64 this_max = physical_type == parquet::Type::INT32 ? Int64(decodeLE(encoded_max)) : decodeLE(encoded_max); + acc.min_int = std::min(acc.min_int, this_min); + acc.max_int = std::max(acc.max_int, this_max); + } + else + { + acc.min_str = std::min(acc.min_str, encoded_min); + acc.max_str = std::max(acc.max_str, encoded_max); + } + return acc; +} + +/// Read the parquet footer of one file and derive its Iceberg statistics and per-column bounds. +AnalyzedFile analyzeFile( + const String & registration_path, + const IcebergPathResolver & resolver, + const ObjectStoragePtr & object_storage, + const std::unordered_map & column_path_to_field_id, + const ContextPtr & context, + const FormatSettings & format_settings) +{ + AnalyzedFile result; + result.registration_path = registration_path; + + IcebergPathFromMetadata metadata_path = IcebergPathFromMetadata::deserialize(registration_path); + result.storage_path = resolver.resolve(metadata_path); + + auto object_metadata = object_storage->tryGetObjectMetadata(result.storage_path, /* with_tags */ false); + if (!object_metadata.has_value()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "add_files: file {} does not exist", registration_path); + result.stats.file_size_in_bytes = static_cast(object_metadata->size_bytes); + + std::shared_ptr parquet_metadata; + try + { + std::atomic is_stopped = 0; + auto buffer = object_storage->readObject(StoredObject(result.storage_path), context->getReadSettings()); + auto arrow_file = asArrowFile(*buffer, format_settings, is_stopped, "Parquet", PARQUET_MAGIC_BYTES, /* avoid_buffering */ true); + parquet_metadata = parquet::ReadMetaData(arrow_file); + } + catch (...) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "add_files: failed to read the parquet footer of {} ({}). The file may be truncated " + "or still being written by another process; registering such a file is the caller's responsibility. " + "Original error: {}", + registration_path, result.storage_path, getCurrentExceptionMessage(false)); + } + + result.stats.record_count = parquet_metadata->num_rows(); + + std::unordered_map column_sizes; + std::unordered_map null_value_counts; + std::unordered_map bounds; + + for (int rg = 0; rg < parquet_metadata->num_row_groups(); ++rg) + { + auto row_group = parquet_metadata->RowGroup(rg); + for (int col = 0; col < row_group->num_columns(); ++col) + { + auto column = row_group->ColumnChunk(col); + String dot_path = column->path_in_schema()->ToDotString(); + auto field_it = column_path_to_field_id.find(dot_path); + if (field_it == column_path_to_field_id.end()) + continue; + Int32 field_id = static_cast(field_it->second); + + column_sizes[field_id] += column->total_compressed_size(); + if (column->is_stats_set()) + { + auto stats = column->statistics(); + if (stats && stats->HasNullCount()) + null_value_counts[field_id] += stats->null_count(); + } + bounds[field_id] = aggregateColumnBounds(*column, std::move(bounds[field_id])); + } + } + + for (const auto & [field_id, size] : column_sizes) + result.stats.column_sizes.emplace_back(field_id, size); + for (const auto & [field_id, count] : null_value_counts) + result.stats.null_value_counts.emplace_back(field_id, count); + for (const auto & [field_id, bound] : bounds) + { + if (!bound.has_bounds) + continue; + if (bound.kind == SourceValueKind::Integer) + { + /// Pass the parquet plain encoding through: both parquet statistics and Iceberg + /// bounds store fixed-width numerics little-endian, with the width of the parquet + /// physical type (INT32 -> 4 bytes, INT64 -> 8 bytes). + std::vector lower_bytes; + std::vector upper_bytes; + if (bound.int_is_32bit) + { + Int32 lo = static_cast(bound.min_int); + Int32 hi = static_cast(bound.max_int); + lower_bytes.resize(sizeof(Int32)); + upper_bytes.resize(sizeof(Int32)); + memcpy(lower_bytes.data(), &lo, sizeof(Int32)); + memcpy(upper_bytes.data(), &hi, sizeof(Int32)); + } + else + { + Int64 lo = bound.min_int; + Int64 hi = bound.max_int; + lower_bytes.resize(sizeof(Int64)); + upper_bytes.resize(sizeof(Int64)); + memcpy(lower_bytes.data(), &lo, sizeof(Int64)); + memcpy(upper_bytes.data(), &hi, sizeof(Int64)); + } + result.stats.lower_bounds.emplace_back(field_id, std::move(lower_bytes)); + result.stats.upper_bounds.emplace_back(field_id, std::move(upper_bytes)); + } + else + { + std::vector lower_bytes(bound.min_str.begin(), bound.min_str.end()); + std::vector upper_bytes(bound.max_str.begin(), bound.max_str.end()); + result.stats.lower_bounds.emplace_back(field_id, std::move(lower_bytes)); + result.stats.upper_bounds.emplace_back(field_id, std::move(upper_bytes)); + } + } + + result.bounds_by_field_id = std::move(bounds); + return result; +} + +/// The value one partition field takes for a file, proven identical for every row in the file. +struct PartitionFieldValue +{ + String column_name; + Field value; /// Null when the transform is void or the column has only nulls + DataTypePtr type; /// ClickHouse type used for manifest tuple serialization +}; + +/// Derive the partition tuple of one file from its per-column bounds and prove that every row +/// in the file belongs to that single partition. Fails with an error naming the file and the +/// offending column when a file spans partitions. +PartitionFieldValue derivePartitionField( + const Poco::JSON::Object::Ptr & spec_field, + const AnalyzedFile & file, + const std::unordered_map & source_columns, + const std::unordered_map & iceberg_type_by_field_id) +{ + Int64 source_id = spec_field->getValue(Iceberg::f_source_id); + String transform = spec_field->getValue(Iceberg::f_transform); + String field_name = spec_field->getValue(Iceberg::f_name); + + PartitionFieldValue result; + result.column_name = field_name; + + auto source_it = source_columns.find(source_id); + if (source_it == source_columns.end()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "add_files: partition field '{}' refers to source id {} which is not in the current schema", field_name, source_id); + const NameAndTypePair & source_column = source_it->second; + + if (transform == "void") + { + result.value = Null(); + result.type = source_column.type; + return result; + } + + /// Parse bucket[N] / truncate[N]. + auto bracket = transform.find('['); + String transform_name = bracket == String::npos ? transform : transform.substr(0, bracket); + Int64 transform_argument = 0; + if (bracket != String::npos) + transform_argument = std::stoll(transform.substr(bracket + 1, transform.size() - bracket - 2)); + + if (transform_name == "bucket") + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, + "add_files: partition field '{}' uses the bucket transform, which cannot be validated from " + "file statistics alone (two different values may share a bucket with a third outside it). " + "Files for bucket-partitioned tables are not supported yet", field_name); + + auto bounds_it = file.bounds_by_field_id.find(static_cast(source_id)); + if (bounds_it == file.bounds_by_field_id.end() || !bounds_it->second.has_bounds) + { + /// Column absent from the file's statistics (e.g. all values are null): the partition + /// value is null. Note this trusts the file's null-only stats; all-null columns carry + /// no min/max by definition. + result.value = Null(); + result.type = transform_name == "identity" || transform_name == "truncate" ? source_column.type : DataTypePtr(std::make_shared()); + return result; + } + const ColumnBounds & bounds = bounds_it->second; + + auto iceberg_type_it = iceberg_type_by_field_id.find(source_id); + const String iceberg_type = iceberg_type_it != iceberg_type_by_field_id.end() ? iceberg_type_it->second : ""; + bool is_timestamp = iceberg_type.starts_with("timestamp"); + bool is_date = iceberg_type == "date"; + + if (transform_name == "identity") + { + bool equal = bounds.kind == SourceValueKind::Integer ? bounds.min_int == bounds.max_int : bounds.min_str == bounds.max_str; + if (!equal) + throw Exception(ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "add_files: file {} spans more than one partition: column '{}' has distinct min and max values", + file.registration_path, source_column.name); + if (bounds.kind == SourceValueKind::Integer) + { + if (is_date) + { + /// Mirrors the native writer: identity Date partition values are the raw day + /// number promoted to UInt64 (ColumnVector -> Field). + result.value = Field(UInt64(bounds.min_int)); + } + else if (is_timestamp) + { + /// Mirrors the native writer: a DateTime64 partition column carries its raw + /// decimal value (scale <= 6, i.e. microseconds for Iceberg timestamp types). + auto scale = getDecimalScale(*removeNullable(source_column.type)); + result.value = Field(DecimalField(Decimal64(bounds.min_int), scale)); + } + else + { + result.value = Field(bounds.min_int); + } + } + else + { + result.value = Field(bounds.min_str); + } + result.type = source_column.type; + return result; + } + + if (bounds.kind != SourceValueKind::Integer && transform_name != "truncate") + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "add_files: partition field '{}' applies transform '{}' to a non-numeric column '{}'", + field_name, transform, source_column.name); + + auto days_of = [&](Int64 v) { return is_timestamp ? floorDiv(v, 86400000000LL) : v; }; + auto year_of = [&](Int64 v) + { + Int64 y; + unsigned m, d; + civilFromDays(days_of(v), y, m, d); + return y - 1970; + }; + auto month_of = [&](Int64 v) + { + Int64 y; + unsigned m, d; + civilFromDays(days_of(v), y, m, d); + return (y - 1970) * 12 + static_cast(m) - 1; + }; + + Int64 lo = bounds.min_int; + Int64 hi = bounds.max_int; + Int64 lo_transformed = 0; + Int64 hi_transformed = 0; + + if (transform_name == "year") + { + lo_transformed = year_of(lo); + hi_transformed = year_of(hi); + } + else if (transform_name == "month") + { + lo_transformed = month_of(lo); + hi_transformed = month_of(hi); + } + else if (transform_name == "day" || transform_name == "days" || transform_name == "date" || transform_name == "dates") + { + lo_transformed = days_of(lo); + hi_transformed = days_of(hi); + } + else if (transform_name == "hour" || transform_name == "hours") + { + lo_transformed = floorDiv(lo, 3600000000LL); + hi_transformed = floorDiv(hi, 3600000000LL); + } + else if (transform_name == "truncate") + { + if (bounds.kind == SourceValueKind::String) + { + String lo_trunc = bounds.min_str.substr(0, static_cast(transform_argument)); + String hi_trunc = bounds.max_str.substr(0, static_cast(transform_argument)); + if (lo_trunc != hi_trunc) + throw Exception(ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "add_files: file {} spans more than one partition: truncated values of column '{}' differ", + file.registration_path, source_column.name); + result.value = Field(lo_trunc); + result.type = source_column.type; + return result; + } + lo_transformed = lo - ((lo % transform_argument + transform_argument) % transform_argument); + hi_transformed = hi - ((hi % transform_argument + transform_argument) % transform_argument); + } + else + { + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, + "add_files: partition field '{}' uses unsupported transform '{}'", field_name, transform); + } + + if (lo_transformed != hi_transformed) + throw Exception(ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "add_files: file {} spans more than one partition: column '{}' maps to different '{}' buckets " + "(min {} -> {}, max {} -> {})", + file.registration_path, source_column.name, transform_name, lo, lo_transformed, hi, hi_transformed); + + result.value = Field(lo_transformed); + result.type = transform_name == "truncate" ? source_column.type : DataTypePtr(std::make_shared()); + return result; +} + +String partitionKeyToString(const std::vector & values) +{ + WriteBufferFromOwnString buf; + for (const auto & value : values) + writeString(applyVisitor(FieldVisitorToString(), value), buf); + return buf.str(); +} + +/// Collect the paths of all data files live in the current snapshot, for duplicate detection. +std::unordered_set listLiveDataFiles( + Poco::JSON::Object::Ptr metadata, + const IcebergPathResolver & resolver, + const ObjectStoragePtr & object_storage, + const FormatSettings & format_settings) +{ + std::unordered_set live_paths; + if (!metadata->has(Iceberg::f_current_snapshot_id) || metadata->isNull(Iceberg::f_current_snapshot_id)) + return live_paths; + + Int64 current_snapshot_id = metadata->getValue(Iceberg::f_current_snapshot_id); + if (current_snapshot_id < 0) + return live_paths; + String manifest_list; + for (const auto & snapshot_var : *metadata->getArray(Iceberg::f_snapshots)) + { + auto snapshot = snapshot_var.extract(); + if (snapshot->getValue(Iceberg::f_metadata_snapshot_id) == current_snapshot_id) + { + manifest_list = snapshot->getValue(Iceberg::f_manifest_list); + break; + } + } + if (manifest_list.empty()) + return live_paths; + + auto read_avro = [&](const String & path) + { + auto buffer = object_storage->readObject( + StoredObject(resolver.resolve(IcebergPathFromMetadata::deserialize(path))), ReadSettings{}); + return AvroForIcebergDeserializer(std::move(buffer), IcebergPathFromMetadata::deserialize(path), format_settings); + }; + + auto manifest_list_reader = read_avro(manifest_list); + for (size_t i = 0; i < manifest_list_reader.rows(); ++i) + { + /// Only data manifests (content == 0) hold data files. + if (manifest_list_reader.getValueFromRowByName(i, Iceberg::f_content, TypeIndex::Int32).safeGet() != 0) + continue; + String manifest_path = manifest_list_reader.getValueFromRowByName(i, Iceberg::f_manifest_path).safeGet(); + auto manifest_reader = read_avro(manifest_path); + for (size_t j = 0; j < manifest_reader.rows(); ++j) + { + auto entry = manifest_reader.getParsedManifestFileEntry(j); + if (entry->status != ManifestEntryStatus::DELETED) + live_paths.insert(entry->file_path_key.serialize()); + } + } + return live_paths; +} + +Pipe resultToPipe(const AddFilesResult & result) +{ + Block header{ + ColumnWithTypeAndName(std::make_shared(), "metric_name"), + ColumnWithTypeAndName(std::make_shared(), "metric_value"), + }; + + MutableColumns columns = header.cloneEmptyColumns(); + auto add = [&](const char * name, Int64 value) + { + columns[0]->insert(String(name)); + columns[1]->insert(value); + }; + + add("files_registered", result.files_registered); + add("rows_registered", result.rows_registered); + add("bytes_registered", result.bytes_registered); + add("partitions_touched", result.partitions_touched); + add("manifests_written", result.manifests_written); + + const size_t rows = columns[0]->size(); + Chunk chunk(std::move(columns), rows); + return Pipe(std::make_shared(std::make_shared(std::move(header)), std::move(chunk))); +} + +} + +AddFilesResult addFiles( + const std::vector & raw_paths, + bool check_duplicate_files, + bool allow_missing_columns, + ContextPtr context, + ObjectStoragePtr object_storage, + const DataLakeStorageSettings & data_lake_settings, + const PersistentTableComponents & persistent_components, + const String & write_format, + std::shared_ptr catalog, + const String & table_name, + ExternalStorageCache & external_storages) +{ + if (Poco::toLower(write_format) != "parquet") + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, + "add_files currently supports only Parquet data files, the table's write format is '{}'", write_format); + + const auto & resolver = persistent_components.path_resolver; + LoggerPtr log = getLogger("IcebergAddFiles"); + FormatSettings format_settings = getFormatSettings(context); + + FileNamesGenerator filename_generator( + resolver.getTableLocation(), + (catalog != nullptr && catalog->isTransactional()), + persistent_components.metadata_compression_method, + write_format); + + auto [last_version, metadata_path, compression_method] = getLatestOrExplicitMetadataFileAndVersion( + object_storage, + persistent_components.table_path, + data_lake_settings, + persistent_components.metadata_cache, + context, + getLogger("IcebergAddFiles").get(), + persistent_components.table_uuid, + persistent_components.metadata_compression_method, + true); + filename_generator.setVersion(last_version + 1); + + Poco::JSON::Object::Ptr metadata = getMetadataJSONObject( + metadata_path, + object_storage, + persistent_components.metadata_cache, + context, + log, + compression_method, + persistent_components.table_uuid); + + const Int64 original_schema_id = metadata->getValue(Iceberg::f_current_schema_id); + const Int64 partition_spec_id = metadata->getValue(Iceberg::f_default_spec_id); + Poco::JSON::Object::Ptr partition_spec = lookupPartitionSpec(metadata, partition_spec_id); + Poco::JSON::Object::Ptr current_schema = lookupSchema(metadata, original_schema_id); + Poco::JSON::Array::Ptr schema_fields = current_schema->getArray(Iceberg::f_fields); + + /// Column path -> field id and back, over all (including nested) leaf fields. + std::unordered_map column_path_to_field_id = IcebergSchemaProcessor::traverseSchema(schema_fields); + std::unordered_map field_id_to_column_path; + for (const auto & [path, id] : column_path_to_field_id) + field_id_to_column_path.emplace(id, path); + + /// Iceberg primitive type per field id (for date/timestamp transform scaling). + std::unordered_map iceberg_type_by_field_id; + { + std::function collect_types; + collect_types = [&](Poco::JSON::Object::Ptr field, const String & prefix) + { + String name = prefix.empty() ? field->getValue(Iceberg::f_name) : prefix + "." + field->getValue(Iceberg::f_name); + Int64 id = field->getValue(Iceberg::f_id); + auto type_var = field->get(Iceberg::f_type); + if (type_var.isString()) + { + iceberg_type_by_field_id[id] = type_var.extract(); + } + else if (type_var.isStruct()) + { + auto type_obj = type_var.extract(); + String kind = type_obj->getValue(Iceberg::f_type); + if (kind == "struct") + { + for (const auto & nested : *type_obj->getArray(Iceberg::f_fields)) + collect_types(nested.extract(), name); + } + } + }; + for (const auto & field_var : *schema_fields) + collect_types(field_var.extract(), ""); + } + + /// Source columns for partition fields, from the ClickHouse schema (top-level fields only). + /// The schema processor is populated lazily (e.g. by reads or native writes); on a freshly + /// created table this add_files call may be the first operation, so register the current + /// schema ourselves first. + if (!persistent_components.schema_processor->hasClickHouseTableSchemaById(static_cast(original_schema_id))) + persistent_components.schema_processor->addIcebergTableSchema(current_schema, context); + auto clickhouse_schema = persistent_components.schema_processor->getClickHouseTableSchemaById(static_cast(original_schema_id)); + Block sample_block; + for (const auto & name_and_type : *clickhouse_schema) + sample_block.insert(ColumnWithTypeAndName(name_and_type.type, name_and_type.name)); + SharedHeader shared_sample_block = std::make_shared(sample_block); + + std::unordered_map source_columns; + Poco::JSON::Array::Ptr spec_fields = partition_spec->getArray(Iceberg::f_fields); + for (const auto & spec_field_var : *spec_fields) + { + auto spec_field = spec_field_var.extract(); + Int64 source_id = spec_field->getValue(Iceberg::f_source_id); + auto path_it = field_id_to_column_path.find(source_id); + if (path_it == field_id_to_column_path.end()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "add_files: partition spec refers to source id {} which is not in the current schema", source_id); + if (path_it->second.find('.') != String::npos) + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, + "add_files: partition source column '{}' is nested; only top-level partition columns are supported", path_it->second); + auto type_it = std::find_if(clickhouse_schema->begin(), clickhouse_schema->end(), + [&](const NameAndTypePair & nt) { return nt.name == path_it->second; }); + if (type_it == clickhouse_schema->end()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Column {} not found in ClickHouse schema", path_it->second); + source_columns.emplace(source_id, *type_it); + } + + /// Expand trailing-slash paths. Registration paths are kept exactly as given (or produced + /// by expansion): they must be in metadata form, i.e. prefixed by the table location, which + /// is the same form native writes register in manifests. + std::vector paths = expandPaths(raw_paths, resolver, object_storage, log); + for (const auto & path : paths) + { + if (!endsWithParquet(path)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "add_files: only .parquet files can be registered, got '{}'", path); + } + + /// Read every file once: footer statistics + bounds. + std::vector analyzed_files; + analyzed_files.reserve(paths.size()); + for (const auto & path : paths) + analyzed_files.push_back(analyzeFile(path, resolver, object_storage, column_path_to_field_id, context, format_settings)); + + /// Validate the file's columns against the table schema. + if (!allow_missing_columns) + { + for (const auto & file : analyzed_files) + { + for (const auto & [field_path, field_id] : column_path_to_field_id) + { + if (iceberg_type_by_field_id.contains(field_id) && !file.bounds_by_field_id.contains(static_cast(field_id))) + { + /// Column missing from the file entirely (no statistics either way). + throw Exception(ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "add_files: file {} lacks column '{}' present in the table schema; " + "pass allow_missing_columns = 1 to register it anyway (values will read as the column default)", + file.registration_path, field_path); + } + } + } + } + + /// Derive (and validate) the partition tuple of every file, then group files by tuple. + struct PartitionGroup + { + std::vector partition_columns; + std::vector partition_values; + DataTypes partition_types; + std::vector file_indices; + }; + std::map groups; + for (size_t i = 0; i < analyzed_files.size(); ++i) + { + std::vector tuple; + std::vector names; + DataTypes types; + for (const auto & spec_field_var : *spec_fields) + { + auto field_value = derivePartitionField( + spec_field_var.extract(), analyzed_files[i], source_columns, iceberg_type_by_field_id); + names.push_back(field_value.column_name); + tuple.push_back(field_value.value); + types.push_back(field_value.type); + } + String key = partitionKeyToString(tuple); + auto [it, inserted] = groups.try_emplace(key); + if (inserted) + { + it->second.partition_columns = names; + it->second.partition_values = tuple; + it->second.partition_types = types; + } + it->second.file_indices.push_back(i); + } + + /// Duplicate detection against the current snapshot and inside the input set. + if (check_duplicate_files) + { + std::unordered_set seen; + for (const auto & file : analyzed_files) + { + if (!seen.insert(file.registration_path).second) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "add_files: file '{}' is listed twice in the input", file.registration_path); + } + auto live_files = listLiveDataFiles(metadata, resolver, object_storage, format_settings); + for (const auto & file : analyzed_files) + { + if (live_files.contains(file.registration_path)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "add_files: file '{}' is already registered in the current snapshot", file.registration_path); + } + } + + /// Commit: one manifest per partition tuple, one manifest list, one append snapshot. + AddFilesResult result; + result.partitions_touched = static_cast(groups.size()); + + size_t retry = 0; + static constexpr size_t max_retries = 100; + while (true) + { + auto metadata_info = filename_generator.generateMetadataPathWithInfo(); + const auto storage_metadata_name = resolver.resolve(metadata_info.path); + + Int64 parent_snapshot = -1; + if (metadata->has(Iceberg::f_current_snapshot_id) && !metadata->isNull(Iceberg::f_current_snapshot_id)) + parent_snapshot = metadata->getValue(Iceberg::f_current_snapshot_id); + + Int64 total_files = 0; + Int64 total_rows = 0; + Int64 total_bytes = 0; + for (const auto & file : analyzed_files) + { + ++total_files; + total_rows += file.stats.record_count; + total_bytes += file.stats.file_size_in_bytes; + } + + auto [new_snapshot, manifest_list_path] = MetadataGenerator(metadata).generateNextMetadata( + filename_generator, metadata_info.path, parent_snapshot, + total_files, total_rows, total_bytes, static_cast(groups.size()), + /* added_delete_files */ 0, /* num_deleted_rows */ 0); + const auto storage_manifest_list_name = resolver.resolve(manifest_list_path); + + std::vector manifest_paths; + std::vector manifest_lengths; + std::vector manifest_row_counts; + std::vector>> manifest_partition_summaries; + bool published = false; + + auto cleanup = [&](bool retry_because_of_metadata_conflict) + { + for (size_t i = 0; i < manifest_paths.size(); ++i) + object_storage->removeObjectIfExists(StoredObject(resolver.resolve(manifest_paths[i]))); + object_storage->removeObjectIfExists(StoredObject(storage_manifest_list_name)); + + if (retry_because_of_metadata_conflict) + { + auto [new_version, new_metadata_path, new_compression] = getLatestOrExplicitMetadataFileAndVersion( + object_storage, + persistent_components.table_path, + data_lake_settings, + persistent_components.metadata_cache, + context, + getLogger("IcebergAddFiles").get(), + persistent_components.table_uuid, + persistent_components.metadata_compression_method, + true); + filename_generator.setVersion(new_version + 1); + metadata = getMetadataJSONObject( + new_metadata_path, object_storage, persistent_components.metadata_cache, + context, log, new_compression, persistent_components.table_uuid); + + if (metadata->getValue(Iceberg::f_current_schema_id) != original_schema_id) + throw Exception(ErrorCodes::METADATA_MISMATCH, + "Table schema changed during add_files; restart the command"); + if (metadata->getValue(Iceberg::f_default_spec_id) != partition_spec_id) + throw Exception(ErrorCodes::METADATA_MISMATCH, + "Table partition spec changed during add_files; restart the command"); + } + }; + + try + { + /// Write one manifest per partition tuple. + for (const auto & [key, group] : groups) + { + auto manifest_path = filename_generator.generateManifestEntryName(); + auto storage_manifest_name = resolver.resolve(manifest_path); + auto buffer = object_storage->writeObject( + StoredObject(storage_manifest_name), WriteMode::Rewrite, std::nullopt, + DBMS_DEFAULT_BUFFER_SIZE, context->getWriteSettings()); + + std::vector group_paths; + std::vector group_row_counts; + std::vector group_byte_counts; + std::vector group_stats; + Int64 group_rows = 0; + for (size_t idx : group.file_indices) + { + group_paths.push_back(IcebergPathFromMetadata::deserialize(analyzed_files[idx].registration_path)); + group_row_counts.push_back(static_cast(analyzed_files[idx].stats.record_count)); + group_byte_counts.push_back(static_cast(analyzed_files[idx].stats.file_size_in_bytes)); + group_stats.push_back(analyzed_files[idx].stats); + group_rows += analyzed_files[idx].stats.record_count; + } + + generateManifestFile( + metadata, + group.partition_columns, + group.partition_values, + group.partition_types, + group_paths, + group_row_counts, + group_byte_counts, + /* data_file_statistics */ std::nullopt, + shared_sample_block, + new_snapshot, + write_format, + partition_spec, + partition_spec_id, + *buffer, + FileContentType::DATA, + /* user_defined_sequence_number */ std::nullopt, + /* user_defined_snapshot_id */ std::nullopt, + /* data_file_formats */ {}, + /* per_file_statistics */ {}, + /* data_file_sort_order_ids */ {}, + /* per_file_entry_lineage */ {}, + /* schema_to_serialize */ nullptr, + /* per_file_fresh_statistics */ nullptr, + /* per_file_stats */ group_stats); + buffer->finalize(); + + manifest_paths.push_back(manifest_path); + manifest_lengths.push_back(static_cast(buffer->count())); + manifest_row_counts.push_back(group_rows); + + std::vector> summary; + summary.reserve(group.partition_values.size()); + for (size_t i = 0; i < group.partition_values.size(); ++i) + summary.emplace_back(group.partition_values[i], group.partition_types[i]); + manifest_partition_summaries.push_back(std::move(summary)); + } + + { + auto buffer_manifest_list = object_storage->writeObject( + StoredObject(storage_manifest_list_name), WriteMode::Rewrite, std::nullopt, + DBMS_DEFAULT_BUFFER_SIZE, context->getWriteSettings()); + generateManifestList( + resolver, metadata, object_storage, external_storages, context, + manifest_paths, new_snapshot, manifest_lengths, *buffer_manifest_list, + FileContentType::DATA, /* use_previous_snapshots */ true, + /* per_entry_content_types */ {}, + /* entry_counts */ {}, + /* carry_forward_manifest_paths */ {}, + /* entry_partition_spec_ids */ {}, + /* entry_partition_summaries */ manifest_partition_summaries, + /* previous_manifest_actions */ {}, + /* entry_row_counts */ manifest_row_counts); + buffer_manifest_list->finalize(); + } + + { + std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + Poco::JSON::Stringifier::stringify(metadata, oss, 4); + std::string json_representation = removeEscapedSlashes(oss.str()); + + auto hint_path = filename_generator.generateVersionHint(); + if (!writeMetadataFileAndVersionHint( + resolver, metadata_info, json_representation, hint_path, object_storage, context, + data_lake_settings[DataLakeStorageSetting::iceberg_use_version_hint])) + { + cleanup(true); + if (++retry >= max_retries) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "add_files: too many metadata conflicts, giving up"); + continue; + } + + if (catalog) + { + auto catalog_filename = resolver.resolveForCatalog(metadata_info.path); + const auto & [namespace_name, table] = DataLake::parseTableName(table_name); + if (!catalog->updateMetadata(namespace_name, table, catalog_filename, new_snapshot, context->getForwardedAuthToken())) + { + cleanup(true); + if (++retry >= max_retries) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "add_files: too many catalog conflicts, giving up"); + continue; + } + } + published = true; + } + + persistent_components.invalidateMetadataCache(); + } + catch (...) + { + if (published) + { + tryLogCurrentException(log, + "Post-publish work failed after Iceberg snapshot was committed by add_files; " + "skipping manifest cleanup to preserve the published snapshot"); + } + else + { + cleanup(false); + } + throw; + } + + result.files_registered = total_files; + result.rows_registered = total_rows; + result.bytes_registered = total_bytes; + result.manifests_written = static_cast(manifest_paths.size()); + return result; + } +} + +Pipe executeAddFiles( + const ASTPtr & args, + ContextPtr context, + ObjectStoragePtr object_storage, + const DataLakeStorageSettings & data_lake_settings, + const PersistentTableComponents & persistent_components, + const String & write_format, + std::shared_ptr catalog, + const String & table_name, + ExternalStorageCache & external_storages) +{ + auto parsed = makeSchema().parse(args); + std::vector paths = parsePathsArgument(parsed.get("paths")); + bool check_duplicates = parsed.getAs("check_duplicate_files") != 0; + bool allow_missing = parsed.getAs("allow_missing_columns") != 0; + + auto result = addFiles( + paths, check_duplicates, allow_missing, + context, object_storage, data_lake_settings, persistent_components, + write_format, catalog, table_name, external_storages); + + return resultToPipe(result); +} + +} +} + +#endif diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/AddFilesExecute.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/AddFilesExecute.h new file mode 100644 index 0000000000000..0c3248947cf0e --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/AddFilesExecute.h @@ -0,0 +1,58 @@ +#pragma once + +#include "config.h" + +#if USE_AVRO + +#include +#include +#include +#include +#include +#include +#include + +namespace DB::Iceberg +{ + +/// Result of an add_files execution, surfaced as (metric_name, metric_value) rows. +struct AddFilesResult +{ + Int64 files_registered = 0; + Int64 rows_registered = 0; + Int64 bytes_registered = 0; + Int64 partitions_touched = 0; + Int64 manifests_written = 0; +}; + +/// Register existing Parquet files as a new append snapshot, like iceberg-java's +/// `add_files`. Each file is read once (footer only) to derive its record count, +/// column sizes, null counts and bounds, and to prove that it belongs to a single +/// partition of the table's current default partition spec. +AddFilesResult addFiles( + const std::vector & paths, + bool check_duplicate_files, + bool allow_missing_columns, + ContextPtr context, + ObjectStoragePtr object_storage, + const DataLakeStorageSettings & data_lake_settings, + const PersistentTableComponents & persistent_table_components, + const String & write_format, + std::shared_ptr catalog, + const String & table_name, + ExternalStorageCache & external_storages); + +Pipe executeAddFiles( + const ASTPtr & args, + ContextPtr context, + ObjectStoragePtr object_storage, + const DataLakeStorageSettings & data_lake_settings, + const PersistentTableComponents & persistent_components, + const String & write_format, + std::shared_ptr catalog, + const String & table_name, + ExternalStorageCache & external_storages); + +} + +#endif diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp index 1e4ca36843d3f..742dc201ff4c0 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp @@ -71,6 +71,7 @@ #include #include #include +#include #include #include #include @@ -1055,6 +1056,13 @@ Pipe IcebergMetadata::executeCommand( return Iceberg::executeRemoveOrphanFiles( args, context, object_storage_, data_lake_settings, persistent_components, *external_storages); } + else if (command_name == "add_files") + { + checkTableRootIsQueriedPath("add_files"); + return Iceberg::executeAddFiles( + args, context, object_storage_, data_lake_settings, persistent_components, + write_format, catalog_, storage_id.getTableName(), *external_storages); + } else { throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown EXECUTE command '{}' for Iceberg table", command_name); diff --git a/tests/integration/test_storage_iceberg_with_spark/test_add_files.py b/tests/integration/test_storage_iceberg_with_spark/test_add_files.py new file mode 100644 index 0000000000000..8d5df5e9f1328 --- /dev/null +++ b/tests/integration/test_storage_iceberg_with_spark/test_add_files.py @@ -0,0 +1,310 @@ +import io +import os +import tempfile + +import pytest + +from helpers.iceberg_utils import ( + create_iceberg_table, + get_uuid_str, +) + + +ICEBERG_SETTINGS = { + "allow_insert_into_iceberg": 1, +} + +# Path of the table location in metadata form (with a leading slash); this is the +# form `add_files` expects, the same one native writes register in manifests. +TABLE_PREFIX = "/var/lib/clickhouse/user_files/iceberg_data/default" +# Storage key prefix inside the MinIO bucket (no leading slash). +S3_KEY_PREFIX = "var/lib/clickhouse/user_files/iceberg_data/default" + + +# --------------------------------------------------------------------------- +# Test environment — binds cluster/instance/storage_type/table_name once +# so individual tests only express the add_files-specific logic. +# --------------------------------------------------------------------------- + +class AddFilesTestEnv: + def __init__(self, cluster, storage_type, table_prefix, schema="(x Int32)", format_version=2, partition_by=""): + self.cluster = cluster + self.instance = cluster.instances["node1"] + self.storage_type = storage_type + self.table_name = f"{table_prefix}_{storage_type}_{get_uuid_str()}" + create_iceberg_table( + storage_type, self.instance, self.table_name, + self.cluster, schema, format_version, + partition_by=partition_by, + ) + + # -- paths --------------------------------------------------------------- + + def table_path(self, subdir, filename): + """Metadata-form path of a file below the table root, as passed to add_files.""" + return f"{TABLE_PREFIX}/{self.table_name}/{subdir}/{filename}" + + # -- standalone parquet files --------------------------------------------- + + def write_parquet(self, subdir, filename, select_sql): + """Produce a standalone parquet file (written by ClickHouse itself, so the + footer matches what Iceberg writers produce) at {table}/{subdir}/{filename}.""" + if self.storage_type == "local": + target_dir = f"{TABLE_PREFIX}/{self.table_name}/{subdir}" + self.instance.exec_in_container(["bash", "-c", f"mkdir -p {target_dir}"]) + self.instance.query( + f"INSERT INTO TABLE FUNCTION " + f"file('iceberg_data/default/{self.table_name}/{subdir}/{filename}') " + f"{select_sql} FORMAT Parquet" + ) + else: + # Write the file inside the container and pull it out with a + # binary-safe copy; the host instance dir is not assumed to be + # reachable from the pytest side. + tmp_filename = f"add_files_tmp_{get_uuid_str()}.parquet" + container_path = f"/var/lib/clickhouse/user_files/{tmp_filename}" + self.instance.query( + f"INSERT INTO TABLE FUNCTION " + f"file('{tmp_filename}') " + f"{select_sql} FORMAT Parquet" + ) + host_path = os.path.join(tempfile.gettempdir(), tmp_filename) + self.instance.copy_file_from_container(container_path, host_path) + with open(host_path, "rb") as f: + data = f.read() + os.remove(host_path) + self.instance.exec_in_container(["rm", "-f", container_path]) + key = f"{S3_KEY_PREFIX}/{self.table_name}/{subdir}/{filename}" + self.cluster.minio_client.put_object( + self.cluster.minio_bucket, key, io.BytesIO(data), len(data), + ) + + # -- command execution ----------------------------------------------------- + + def add_files_query(self, paths, check_duplicate_files=None, allow_missing_columns=None): + if isinstance(paths, str): + paths_sql = f"'{paths}'" + else: + paths_sql = "[" + ", ".join(f"'{p}'" for p in paths) + "]" + args = [paths_sql] + if check_duplicate_files is not None: + args.append(f"check_duplicate_files = {check_duplicate_files}") + if allow_missing_columns is not None: + args.append(f"allow_missing_columns = {allow_missing_columns}") + return f"ALTER TABLE {self.table_name} EXECUTE add_files({', '.join(args)});" + + @staticmethod + def parse_metrics(raw): + counts = {} + for line in raw.strip().split("\n"): + if not line: + continue + parts = line.split("\t") + if len(parts) == 2: + counts[parts[0]] = int(parts[1]) + return counts + + def add_files(self, paths, **kwargs): + raw = self.instance.query(self.add_files_query(paths, **kwargs), settings=ICEBERG_SETTINGS) + return self.parse_metrics(raw) + + def add_files_error(self, paths, **kwargs): + return self.instance.query_and_get_error(self.add_files_query(paths, **kwargs), settings=ICEBERG_SETTINGS) + + # -- helpers ---------------------------------------------------------------- + + def insert(self, values): + self.instance.query(f"INSERT INTO {self.table_name} VALUES {values};", settings=ICEBERG_SETTINGS) + + def query(self, sql): + return self.instance.query(sql, settings=ICEBERG_SETTINGS) + + def count(self): + return int(self.query(f"SELECT count() FROM {self.table_name}")) + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +@pytest.mark.parametrize("format_version", [1, 2]) +def test_add_files_basic(started_cluster_iceberg_with_spark, storage_type, format_version): + """Register one external parquet file into an empty table and read it back.""" + env = AddFilesTestEnv(started_cluster_iceberg_with_spark, storage_type, "test_add_basic", format_version=format_version) + + env.write_parquet("data/import", "f1.parquet", "SELECT number + 1 AS x FROM numbers(3)") + counts = env.add_files(env.table_path("data/import", "f1.parquet")) + + assert counts["files_registered"] == 1, f"got {counts}" + assert counts["rows_registered"] == 3, f"got {counts}" + assert counts["manifests_written"] == 1, f"got {counts}" + assert counts["bytes_registered"] > 0, f"got {counts}" + assert env.query(f"SELECT * FROM {env.table_name} ORDER BY x") == "1\n2\n3\n" + + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +def test_add_files_array_form_multiple(started_cluster_iceberg_with_spark, storage_type): + """The array form registers several files in one snapshot.""" + env = AddFilesTestEnv(started_cluster_iceberg_with_spark, storage_type, "test_add_array") + + env.write_parquet("data/import", "f1.parquet", "SELECT number AS x FROM numbers(3)") + env.write_parquet("data/import", "f2.parquet", "SELECT number + 10 AS x FROM numbers(3)") + counts = env.add_files([ + env.table_path("data/import", "f1.parquet"), + env.table_path("data/import", "f2.parquet"), + ]) + + assert counts["files_registered"] == 2, f"got {counts}" + assert counts["rows_registered"] == 6, f"got {counts}" + assert env.count() == 6 + assert env.query(f"SELECT * FROM {env.table_name} ORDER BY x") == "0\n1\n2\n10\n11\n12\n" + + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +def test_add_files_onto_nonempty_table(started_cluster_iceberg_with_spark, storage_type): + """Registration appends a snapshot on top of natively-written data.""" + env = AddFilesTestEnv(started_cluster_iceberg_with_spark, storage_type, "test_add_nonempty") + + env.insert("(1), (2)") + env.write_parquet("data/import", "f1.parquet", "SELECT number + 100 AS x FROM numbers(3)") + counts = env.add_files(env.table_path("data/import", "f1.parquet")) + + assert counts["files_registered"] == 1, f"got {counts}" + assert env.count() == 5 + assert env.query(f"SELECT * FROM {env.table_name} ORDER BY x") == "1\n2\n100\n101\n102\n" + + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +def test_add_files_trailing_slash_expansion(started_cluster_iceberg_with_spark, storage_type): + """A path ending with '/' registers every *.parquet below it, skipping hidden files.""" + env = AddFilesTestEnv(started_cluster_iceberg_with_spark, storage_type, "test_add_expand") + + env.write_parquet("data/import", "f1.parquet", "SELECT 1 AS x") + env.write_parquet("data/import", "f2.parquet", "SELECT 2 AS x") + env.write_parquet("data/import", ".hidden.parquet", "SELECT 3 AS x") + + counts = env.add_files(env.table_path("data/import", "")) + + assert counts["files_registered"] == 2, f"got {counts}" + assert env.count() == 2 + assert env.query(f"SELECT * FROM {env.table_name} ORDER BY x") == "1\n2\n" + + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +def test_add_files_partitioned_identity(started_cluster_iceberg_with_spark, storage_type): + """Single-partition files register into an identity-partitioned table; a file + spanning two partition values is rejected.""" + env = AddFilesTestEnv( + started_cluster_iceberg_with_spark, storage_type, "test_add_identity", + partition_by="identity(x)", + ) + + env.write_parquet("data/import", "f7.parquet", "SELECT 7 AS x FROM numbers(3)") + counts = env.add_files(env.table_path("data/import", "f7.parquet")) + assert counts["files_registered"] == 1, f"got {counts}" + assert counts["partitions_touched"] == 1, f"got {counts}" + + env.write_parquet("data/import", "f8.parquet", "SELECT 8 AS x FROM numbers(2)") + counts = env.add_files(env.table_path("data/import", "f8.parquet")) + assert counts["partitions_touched"] == 1, f"got {counts}" + assert env.query(f"SELECT * FROM {env.table_name} ORDER BY x") == "7\n7\n7\n8\n8\n" + + env.write_parquet("data/import", "mixed.parquet", "SELECT 7 AS x UNION ALL SELECT 8 AS x") + error = env.add_files_error(env.table_path("data/import", "mixed.parquet")) + assert "spans more than one partition" in error, f"got {error}" + # The failed registration must not have committed anything. + assert env.count() == 5 + + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +def test_add_files_day_transform(started_cluster_iceberg_with_spark, storage_type): + """The day() transform is validated from footer bounds: same-day file registers, + a file crossing midnight is rejected.""" + env = AddFilesTestEnv( + started_cluster_iceberg_with_spark, storage_type, "test_add_day", + schema="(d DateTime64(6))", + partition_by="toRelativeDayNum(d)", + ) + + env.write_parquet( + "data/import", "same_day.parquet", + "SELECT toDateTime64('2024-01-01 10:00:00', 6) AS d " + "UNION ALL SELECT toDateTime64('2024-01-01 23:00:00', 6) AS d", + ) + counts = env.add_files(env.table_path("data/import", "same_day.parquet")) + assert counts["files_registered"] == 1, f"got {counts}" + assert env.count() == 2 + + env.write_parquet( + "data/import", "two_days.parquet", + "SELECT toDateTime64('2024-01-01 10:00:00', 6) AS d " + "UNION ALL SELECT toDateTime64('2024-01-02 10:00:00', 6) AS d", + ) + error = env.add_files_error(env.table_path("data/import", "two_days.parquet")) + assert "spans more than one partition" in error, f"got {error}" + assert env.count() == 2 + + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +def test_add_files_duplicate_detection(started_cluster_iceberg_with_spark, storage_type): + """Duplicates against the current snapshot and inside the input are rejected by + default; check_duplicate_files = 0 disables the check.""" + env = AddFilesTestEnv(started_cluster_iceberg_with_spark, storage_type, "test_add_dup") + + env.write_parquet("data/import", "f1.parquet", "SELECT 1 AS x") + path = env.table_path("data/import", "f1.parquet") + + counts = env.add_files(path) + assert counts["files_registered"] == 1, f"got {counts}" + + error = env.add_files_error(path) + assert "already registered in the current snapshot" in error, f"got {error}" + + error = env.add_files_error([path, path]) + assert "listed twice" in error, f"got {error}" + + counts = env.add_files(path, check_duplicate_files=0) + assert counts["files_registered"] == 1, f"got {counts}" + assert env.count() == 2 + + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +def test_add_files_allow_missing_columns(started_cluster_iceberg_with_spark, storage_type): + """A file lacking a schema column is rejected by default and readable with + allow_missing_columns = 1 (missing values read as NULL).""" + env = AddFilesTestEnv( + started_cluster_iceberg_with_spark, storage_type, "test_add_missing", + schema="(x Int32, y Nullable(String))", + ) + + env.write_parquet("data/import", "only_x.parquet", "SELECT 1 AS x") + path = env.table_path("data/import", "only_x.parquet") + + error = env.add_files_error(path) + assert "lacks column" in error, f"got {error}" + assert "'y'" in error, f"got {error}" + + counts = env.add_files(path, allow_missing_columns=1) + assert counts["files_registered"] == 1, f"got {counts}" + assert env.query(f"SELECT x, y FROM {env.table_name}") == "1\t\\N\n" + + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +def test_add_files_argument_errors(started_cluster_iceberg_with_spark, storage_type): + """Missing files, non-parquet paths, and bucket-partitioned tables are rejected.""" + env = AddFilesTestEnv(started_cluster_iceberg_with_spark, storage_type, "test_add_args") + + error = env.add_files_error(env.table_path("data", "nonexistent.parquet")) + assert "does not exist" in error, f"got {error}" + + error = env.add_files_error(env.table_path("data", "f1.orc")) + assert ".parquet" in error, f"got {error}" + + env_bucket = AddFilesTestEnv( + started_cluster_iceberg_with_spark, storage_type, "test_add_bucket", + partition_by="icebergBucket(4, x)", + ) + env_bucket.write_parquet("data/import", "f1.parquet", "SELECT 1 AS x") + error = env_bucket.add_files_error(env_bucket.table_path("data/import", "f1.parquet")) + assert "bucket" in error, f"got {error}" From d69bf749a1b0d923f61a4167d3cf6d5ab7928c03 Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Sun, 1 Nov 2026 17:52:37 +0100 Subject: [PATCH 2/9] Refactored to use existing functions in IcebergMetadata --- .../DataLakes/Iceberg/AddFilesExecute.cpp | 78 +------------------ .../Iceberg/AlterDropPartitionExecutor.cpp | 15 ---- .../DataLakes/Iceberg/Compaction.cpp | 22 +----- .../DataLakes/Iceberg/IcebergMetadata.cpp | 12 +-- .../DataLakes/Iceberg/IcebergMetadata.h | 5 ++ .../ObjectStorage/DataLakes/Iceberg/Utils.cpp | 51 ++++++++++++ .../ObjectStorage/DataLakes/Iceberg/Utils.h | 7 ++ 7 files changed, 73 insertions(+), 117 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/AddFilesExecute.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/AddFilesExecute.cpp index d2d735e738621..a3be33646d08f 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/AddFilesExecute.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/AddFilesExecute.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -132,35 +133,6 @@ bool endsWithParquet(const String & path) return Poco::toLower(path).ends_with(".parquet"); } -/// Mirrors the anonymous-namespace helper in IcebergMetadata.cpp: find the partition spec -/// object with the given spec-id inside a metadata JSON document; throws if absent. -Poco::JSON::Object::Ptr lookupPartitionSpec(const Poco::JSON::Object::Ptr & metadata, Int64 spec_id) -{ - auto specs = metadata->getArray(Iceberg::f_partition_specs); - for (size_t i = 0; i < specs->size(); ++i) - { - auto spec = specs->getObject(static_cast(i)); - if (spec->getValue(Iceberg::f_spec_id) == spec_id) - return spec; - } - throw Exception(ErrorCodes::METADATA_MISMATCH, - "Partition spec with id {} not found in table metadata", spec_id); -} - -/// Find the schema object with the given schema-id inside a metadata JSON document; throws if absent. -Poco::JSON::Object::Ptr lookupSchema(const Poco::JSON::Object::Ptr & metadata, Int64 schema_id) -{ - auto schemas = metadata->getArray(Iceberg::f_schemas); - for (size_t i = 0; i < schemas->size(); ++i) - { - auto schema = schemas->getObject(static_cast(i)); - if (schema->getValue(Iceberg::f_schema_id) == schema_id) - return schema; - } - throw Exception(ErrorCodes::METADATA_MISMATCH, - "Schema with id {} not found in table metadata", schema_id); -} - /// Expand trailing-slash paths into concrete *.parquet files below them. std::vector expandPaths( const std::vector & raw_paths, @@ -625,22 +597,7 @@ std::unordered_set listLiveDataFiles( const FormatSettings & format_settings) { std::unordered_set live_paths; - if (!metadata->has(Iceberg::f_current_snapshot_id) || metadata->isNull(Iceberg::f_current_snapshot_id)) - return live_paths; - - Int64 current_snapshot_id = metadata->getValue(Iceberg::f_current_snapshot_id); - if (current_snapshot_id < 0) - return live_paths; - String manifest_list; - for (const auto & snapshot_var : *metadata->getArray(Iceberg::f_snapshots)) - { - auto snapshot = snapshot_var.extract(); - if (snapshot->getValue(Iceberg::f_metadata_snapshot_id) == current_snapshot_id) - { - manifest_list = snapshot->getValue(Iceberg::f_manifest_list); - break; - } - } + String manifest_list = getCurrentManifestListPath(metadata); if (manifest_list.empty()) return live_paths; @@ -757,32 +714,7 @@ AddFilesResult addFiles( field_id_to_column_path.emplace(id, path); /// Iceberg primitive type per field id (for date/timestamp transform scaling). - std::unordered_map iceberg_type_by_field_id; - { - std::function collect_types; - collect_types = [&](Poco::JSON::Object::Ptr field, const String & prefix) - { - String name = prefix.empty() ? field->getValue(Iceberg::f_name) : prefix + "." + field->getValue(Iceberg::f_name); - Int64 id = field->getValue(Iceberg::f_id); - auto type_var = field->get(Iceberg::f_type); - if (type_var.isString()) - { - iceberg_type_by_field_id[id] = type_var.extract(); - } - else if (type_var.isStruct()) - { - auto type_obj = type_var.extract(); - String kind = type_obj->getValue(Iceberg::f_type); - if (kind == "struct") - { - for (const auto & nested : *type_obj->getArray(Iceberg::f_fields)) - collect_types(nested.extract(), name); - } - } - }; - for (const auto & field_var : *schema_fields) - collect_types(field_var.extract(), ""); - } + auto iceberg_type_by_field_id = collectIcebergTypesByFieldId(schema_fields); /// Source columns for partition fields, from the ClickHouse schema (top-level fields only). /// The schema processor is populated lazily (e.g. by reads or native writes); on a freshly @@ -1054,9 +986,7 @@ AddFilesResult addFiles( } { - std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM - Poco::JSON::Stringifier::stringify(metadata, oss, 4); - std::string json_representation = removeEscapedSlashes(oss.str()); + std::string json_representation = stringifyJSON(metadata, 4); auto hint_path = filename_generator.generateVersionHint(); if (!writeMetadataFileAndVersionHint( diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/AlterDropPartitionExecutor.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/AlterDropPartitionExecutor.cpp index f9e4db801455a..937b4eab18e0a 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/AlterDropPartitionExecutor.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/AlterDropPartitionExecutor.cpp @@ -92,21 +92,6 @@ static constexpr auto MAX_TRANSACTION_RETRIES = 100; namespace { -/// Find the partition spec object with the given spec-id inside a metadata JSON document. -/// Throws METADATA_MISMATCH if the spec is not found (indicates metadata/spec-id mismatch). -Poco::JSON::Object::Ptr lookupPartitionSpec(const Poco::JSON::Object::Ptr & meta, Int64 spec_id) -{ - auto specs = meta->getArray(f_partition_specs); - for (size_t i = 0; i < specs->size(); ++i) - { - auto spec = specs->getObject(static_cast(i)); - if (spec->getValue(f_spec_id) == spec_id) - return spec; - } - throw Exception(ErrorCodes::METADATA_MISMATCH, - "Partition spec with id {} not found in table metadata", spec_id); -} - /// One term per partition field in spec order PartitionSpecSignature getPartitionSpecSignature(const Poco::JSON::Array::Ptr & spec_fields) { diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp index c28414b47fc0f..43e5fc5a2d0be 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp @@ -160,23 +160,7 @@ static bool isCurrentManifestListAboveThreshold( { LoggerPtr log = getLogger("IcebergCompaction::isCurrentManifestListAboveThreshold"); - if (!metadata_object->has(Iceberg::f_current_snapshot_id)) - return false; - Int64 current_snapshot_id = metadata_object->getValue(Iceberg::f_current_snapshot_id); - if (current_snapshot_id < 0) - return false; - - String current_manifest_list_path; - auto snapshots = metadata_object->get(Iceberg::f_snapshots).extract(); - for (size_t i = 0; i < snapshots->size(); ++i) - { - const auto snapshot = snapshots->getObject(static_cast(i)); - if (snapshot->getValue(Iceberg::f_metadata_snapshot_id) == current_snapshot_id) - { - current_manifest_list_path = snapshot->getValue(Iceberg::f_manifest_list); - break; - } - } + String current_manifest_list_path = getCurrentManifestListPath(metadata_object); if (current_manifest_list_path.empty()) return false; @@ -981,9 +965,7 @@ static bool writeConsolidatedManifestFile( // Commit: write metadata file with If-None-Match + ETag-based CAS version hint; returns false if another writer claimed this version, so the caller retries. { - std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM - Poco::JSON::Stringifier::stringify(metadata_object, oss, 4); - std::string json_representation = removeEscapedSlashes(oss.str()); + std::string json_representation = stringifyJSON(metadata_object, 4); auto hint_path = generator.generateVersionHint(); LOG_INFO(log, "Committing metadata file: {}", diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp index 742dc201ff4c0..2bcb5d11a82c4 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp @@ -2170,11 +2170,6 @@ namespace FailPoints extern const char iceberg_writes_post_publish_throw[]; } -namespace -{ - -/// Find the partition spec object with the given spec-id inside a metadata JSON document. -/// Throws METADATA_MISMATCH if the spec is not found (indicates metadata/spec-id mismatch). Poco::JSON::Object::Ptr lookupPartitionSpec(const Poco::JSON::Object::Ptr & meta, Int64 spec_id) { auto specs = meta->getArray(Iceberg::f_partition_specs); @@ -2202,6 +2197,9 @@ Poco::JSON::Object::Ptr lookupSchema(const Poco::JSON::Object::Ptr & meta, Int64 "Schema with id {} not found in table metadata", schema_id); } +namespace +{ + /// Derive the Iceberg partition tuple for an exported part from a representative source row. /// The MergeTree `partition.value` is the source partition-key expression result; it is neither /// cast to the destination column types nor expressed through the Iceberg transform, so it must @@ -2481,9 +2479,7 @@ std::optional IcebergMetadata::commitImport } { - std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM - Poco::JSON::Stringifier::stringify(metadata, oss, 4); - std::string json_representation = removeEscapedSlashes(oss.str()); + std::string json_representation = stringifyJSON(metadata, 4); LOG_DEBUG(log, "Writing new metadata file {}", storage_metadata_name); auto hint_path = filename_generator.generateVersionHint(); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h index ab1d0762cafec..9b102d0f8d0e7 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h @@ -334,6 +334,11 @@ class IcebergMetadata : public IDataLakeMetadata std::shared_ptr catalog, const StorageID & storage_id); }; + +/// Find the partition spec / schema object by id inside a metadata JSON document. +/// Throws METADATA_MISMATCH if not found. +Poco::JSON::Object::Ptr lookupPartitionSpec(const Poco::JSON::Object::Ptr & metadata, Int64 spec_id); +Poco::JSON::Object::Ptr lookupSchema(const Poco::JSON::Object::Ptr & metadata, Int64 schema_id); } #endif diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp index 3155e86a5c2f9..1aff7bed89d59 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp @@ -1,7 +1,9 @@ #include +#include #include #include +#include #include #include #include @@ -2237,6 +2239,55 @@ PartitionColumnValues getIdentityPartitionColumnValues( return result; } +std::unordered_map collectIcebergTypesByFieldId(Poco::JSON::Array::Ptr schema_fields) +{ + std::unordered_map result; + + std::function collect_types; + collect_types = [&](Poco::JSON::Object::Ptr field, const String & prefix) + { + String name = prefix.empty() ? field->getValue(Iceberg::f_name) : prefix + "." + field->getValue(Iceberg::f_name); + Int64 id = field->getValue(Iceberg::f_id); + auto type_var = field->get(Iceberg::f_type); + if (type_var.isString()) + { + result[id] = type_var.extract(); + } + else if (type_var.isStruct()) + { + auto type_obj = type_var.extract(); + String kind = type_obj->getValue(Iceberg::f_type); + if (kind == "struct") + { + for (const auto & nested : *type_obj->getArray(Iceberg::f_fields)) + collect_types(nested.extract(), name); + } + } + }; + for (const auto & field_var : *schema_fields) + collect_types(field_var.extract(), ""); + + return result; +} + +String getCurrentManifestListPath(const Poco::JSON::Object::Ptr & metadata) +{ + if (!metadata->has(Iceberg::f_current_snapshot_id) || metadata->isNull(Iceberg::f_current_snapshot_id)) + return {}; + + Int64 current_snapshot_id = metadata->getValue(Iceberg::f_current_snapshot_id); + if (current_snapshot_id < 0) + return {}; + + for (const auto & snapshot_var : *metadata->getArray(Iceberg::f_snapshots)) + { + auto snapshot = snapshot_var.extract(); + if (snapshot->getValue(Iceberg::f_metadata_snapshot_id) == current_snapshot_id) + return snapshot->getValue(Iceberg::f_manifest_list); + } + return {}; +} + } #endif diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h index ceda88fc76e12..fcd8a81036b8c 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h @@ -190,6 +190,13 @@ using PartitionColumnValues = std::vector>; PartitionColumnValues getIdentityPartitionColumnValues( const ProcessedManifestFileEntry & manifest_file_entry, const IcebergSchemaProcessor & schema_processor); + +/// Walk a schema's fields array and return field-id -> raw Iceberg type string +/// (e.g. "date", "timestamptz") for every leaf field, including nested struct members. +std::unordered_map collectIcebergTypesByFieldId(Poco::JSON::Array::Ptr schema_fields); + +/// Returns the manifest-list path of the current snapshot, or empty string if no current snapshot. +String getCurrentManifestListPath(const Poco::JSON::Object::Ptr & metadata); } #endif From 3dba97e5a8e14a93e3bd886ecfec92fb0a1662c2 Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Thu, 12 Nov 2026 22:19:45 +0100 Subject: [PATCH 3/9] Update delta-kernel-rs to include get_domain_metadata_internal --- contrib/delta-kernel-rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/delta-kernel-rs b/contrib/delta-kernel-rs index 88ff5a9474782..955426496cedd 160000 --- a/contrib/delta-kernel-rs +++ b/contrib/delta-kernel-rs @@ -1 +1 @@ -Subproject commit 88ff5a947478215d285783b42aafe6f3d04aca58 +Subproject commit 955426496cedd4496100999b049aa8bd58186eac From 3bc5d8c23521c86fb80410370f77face1a58731a Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Thu, 12 Nov 2026 22:26:05 +0100 Subject: [PATCH 4/9] Revert "Update delta-kernel-rs to include get_domain_metadata_internal" This reverts commit 3dba97e5a8e14a93e3bd886ecfec92fb0a1662c2. --- contrib/delta-kernel-rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/delta-kernel-rs b/contrib/delta-kernel-rs index 955426496cedd..88ff5a9474782 160000 --- a/contrib/delta-kernel-rs +++ b/contrib/delta-kernel-rs @@ -1 +1 @@ -Subproject commit 955426496cedd4496100999b049aa8bd58186eac +Subproject commit 88ff5a947478215d285783b42aafe6f3d04aca58 From 0a184db608dcc19b9aab1225e76b72f64c1e8904 Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Wed, 18 Nov 2026 07:16:25 -0800 Subject: [PATCH 5/9] - Reject parquet columns whose type does not match the Iceberg type exactly (e.g. data for , ms/ns timestamps, , decimals), and reject unsupported Iceberg types, so footer statistics can be used as bounds as they are. - Use the partition only when the source column is absent or all its values are null; reject files without complete min/max statistics. Drop bounds and null counts that do not cover every row group. - Never delete the files of a snapshot that may be visible: with a non-transactional catalog, report a failed catalog update instead of cleaning up and retrying; with a transactional catalog, check whether the catalog applied the snapshot before cleaning up, and do not write the metadata file to storage. - Read the current metadata through the catalog (), and repeat the duplicate check after every retry. - Validate / arguments and handle identity partition values. - Add tests with pyarrow-written files for mismatched types, missing statistics and all-null partitions. --- .../DataLakes/Iceberg/AddFilesExecute.cpp | 334 +++++++++++++++--- .../test_add_files.py | 102 ++++++ 2 files changed, 382 insertions(+), 54 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/AddFilesExecute.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/AddFilesExecute.cpp index a3be33646d08f..cfb52558cf261 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/AddFilesExecute.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/AddFilesExecute.cpp @@ -50,8 +50,10 @@ #include #include +#include #include #include +#include #include #include #include @@ -62,6 +64,7 @@ namespace DB namespace ErrorCodes { extern const int BAD_ARGUMENTS; +extern const int DATALAKE_DATABASE_ERROR; extern const int ICEBERG_SPECIFICATION_VIOLATION; extern const int LOGICAL_ERROR; extern const int METADATA_MISMATCH; @@ -198,6 +201,8 @@ enum class SourceValueKind struct ColumnBounds { bool has_bounds = false; + /// Some row group with non-null values has no min/max, so the bounds do not cover every row. + bool incomplete = false; /// For Integer bounds: whether the parquet physical type is INT32 (4-byte LE serialization) /// rather than INT64 (8-byte). Decided by the physical type, never by the value range. bool int_is_32bit = false; @@ -222,18 +227,76 @@ struct AnalyzedFile String registration_path; /// path form stored in the manifest String storage_path; /// resolved path used for reads IcebergSerializedFileStats stats; + /// Has an entry for every table column present in the file, with or without statistics. std::unordered_map bounds_by_field_id; + /// Field ids whose null count is known for every row group of the file. + std::unordered_map complete_null_counts; }; +/// Check that the parquet column has exactly the representation that the Iceberg type requires, +/// so that its statistics can be used as Iceberg bounds and partition values as they are. +/// Other representations (e.g. decimals, millisecond or nanosecond timestamps, INT96, widening +/// INT32 to long) are rejected rather than converted. +void checkParquetColumnType( + const parquet::ColumnDescriptor & column, const String & iceberg_type, const String & registration_path) +{ + const auto physical_type = column.physical_type(); + const auto & logical_type = column.logical_type(); + const bool no_logical_type = !logical_type || logical_type->is_none(); + + auto is_int_of_width = [&](int bit_width) + { + if (no_logical_type) + return true; + if (!logical_type->is_int()) + return false; + const auto & int_type = static_cast(*logical_type); + return int_type.bit_width() == bit_width && int_type.is_signed(); + }; + + bool matches = false; + if (iceberg_type == "int") + matches = physical_type == parquet::Type::INT32 && is_int_of_width(32); + else if (iceberg_type == "long") + matches = physical_type == parquet::Type::INT64 && is_int_of_width(64); + else if (iceberg_type == "string") + matches = physical_type == parquet::Type::BYTE_ARRAY && (no_logical_type || logical_type->is_string()); + else if (iceberg_type == "date") + matches = physical_type == parquet::Type::INT32 && logical_type && logical_type->is_date(); + else if (iceberg_type == "timestamp" || iceberg_type == "timestamptz") + matches = physical_type == parquet::Type::INT64 && logical_type && logical_type->is_timestamp() + && static_cast(*logical_type).time_unit() == parquet::LogicalType::TimeUnit::MICROS; + else if (iceberg_type == "boolean") + matches = physical_type == parquet::Type::BOOLEAN; + else if (iceberg_type == "float") + matches = physical_type == parquet::Type::FLOAT; + else if (iceberg_type == "double") + matches = physical_type == parquet::Type::DOUBLE; + else + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, + "add_files: column '{}' of file {} has Iceberg type '{}', which add_files does not support yet", + column.path()->ToDotString(), registration_path, iceberg_type); + + if (!matches) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "add_files: column '{}' of file {} is stored as parquet type {} ({}), which does not match the Iceberg type '{}' " + "of the table column", + column.path()->ToDotString(), registration_path, parquet::TypeToString(physical_type), + logical_type ? logical_type->ToString() : "no logical type", iceberg_type); +} + ColumnBounds aggregateColumnBounds( const parquet::ColumnChunkMetaData & column, ColumnBounds acc) { - if (!column.is_stats_set()) - return acc; - auto stats = column.statistics(); + std::shared_ptr stats = column.is_stats_set() ? column.statistics() : nullptr; if (!stats || !stats->HasMinMax()) + { + /// Only a row group whose values are all null may lack min/max. + if (!stats || !stats->HasNullCount() || stats->null_count() != column.num_values()) + acc.incomplete = true; return acc; + } String encoded_min = stats->EncodeMin(); String encoded_max = stats->EncodeMax(); @@ -267,7 +330,7 @@ ColumnBounds aggregateColumnBounds( } if (kind != acc.kind) - return acc; + throw Exception(ErrorCodes::LOGICAL_ERROR, "add_files: statistics of one column have different physical types"); if (kind == SourceValueKind::Integer) { @@ -290,6 +353,7 @@ AnalyzedFile analyzeFile( const IcebergPathResolver & resolver, const ObjectStoragePtr & object_storage, const std::unordered_map & column_path_to_field_id, + const std::unordered_map & iceberg_type_by_field_id, const ContextPtr & context, const FormatSettings & format_settings) { @@ -323,8 +387,26 @@ AnalyzedFile analyzeFile( result.stats.record_count = parquet_metadata->num_rows(); + /// Validate the type of every table column present in the file. Columns whose Iceberg type is not + /// known here (leaves of lists and maps) are registered without bounds. + std::unordered_set columns_with_bounds; + const auto * parquet_schema = parquet_metadata->schema(); + for (int col = 0; col < parquet_schema->num_columns(); ++col) + { + const auto * descriptor = parquet_schema->Column(col); + auto field_it = column_path_to_field_id.find(descriptor->path()->ToDotString()); + if (field_it == column_path_to_field_id.end()) + continue; + auto type_it = iceberg_type_by_field_id.find(field_it->second); + if (type_it == iceberg_type_by_field_id.end()) + continue; + checkParquetColumnType(*descriptor, type_it->second, registration_path); + columns_with_bounds.insert(static_cast(field_it->second)); + } + std::unordered_map column_sizes; std::unordered_map null_value_counts; + std::unordered_set incomplete_null_counts; std::unordered_map bounds; for (int rg = 0; rg < parquet_metadata->num_row_groups(); ++rg) @@ -340,22 +422,32 @@ AnalyzedFile analyzeFile( Int32 field_id = static_cast(field_it->second); column_sizes[field_id] += column->total_compressed_size(); - if (column->is_stats_set()) - { - auto stats = column->statistics(); - if (stats && stats->HasNullCount()) - null_value_counts[field_id] += stats->null_count(); - } - bounds[field_id] = aggregateColumnBounds(*column, std::move(bounds[field_id])); + std::shared_ptr stats = column->is_stats_set() ? column->statistics() : nullptr; + if (stats && stats->HasNullCount()) + null_value_counts[field_id] += stats->null_count(); + else + incomplete_null_counts.insert(field_id); + + if (columns_with_bounds.contains(field_id)) + bounds[field_id] = aggregateColumnBounds(*column, std::move(bounds[field_id])); + else + bounds.try_emplace(field_id); } } for (const auto & [field_id, size] : column_sizes) result.stats.column_sizes.emplace_back(field_id, size); for (const auto & [field_id, count] : null_value_counts) + { + if (incomplete_null_counts.contains(field_id)) + continue; result.stats.null_value_counts.emplace_back(field_id, count); - for (const auto & [field_id, bound] : bounds) + result.complete_null_counts.emplace(field_id, count); + } + for (auto & [field_id, bound] : bounds) { + if (bound.incomplete) + bound.has_bounds = false; if (!bound.has_bounds) continue; if (bound.kind == SourceValueKind::Integer) @@ -441,7 +533,14 @@ PartitionFieldValue derivePartitionField( String transform_name = bracket == String::npos ? transform : transform.substr(0, bracket); Int64 transform_argument = 0; if (bracket != String::npos) - transform_argument = std::stoll(transform.substr(bracket + 1, transform.size() - bracket - 2)); + { + const char * begin = transform.data() + bracket + 1; + const char * end = transform.data() + transform.size() - 1; + auto [ptr, ec] = std::from_chars(begin, end, transform_argument); + if (ec != std::errc() || ptr != end || transform.back() != ']' || transform_argument <= 0) + throw Exception(ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "add_files: partition field '{}' has malformed transform '{}'", field_name, transform); + } if (transform_name == "bucket") throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, @@ -452,9 +551,18 @@ PartitionFieldValue derivePartitionField( auto bounds_it = file.bounds_by_field_id.find(static_cast(source_id)); if (bounds_it == file.bounds_by_field_id.end() || !bounds_it->second.has_bounds) { - /// Column absent from the file's statistics (e.g. all values are null): the partition - /// value is null. Note this trusts the file's null-only stats; all-null columns carry - /// no min/max by definition. + /// The partition value is null only if every row has null in the source column: the column is + /// absent from the file (allowed by `allow_missing_columns`), or the footer proves that all of its + /// values are null. Missing statistics alone prove nothing (writers drop them, e.g. for long values). + if (bounds_it != file.bounds_by_field_id.end()) + { + auto null_count_it = file.complete_null_counts.find(static_cast(source_id)); + if (null_count_it == file.complete_null_counts.end() || null_count_it->second != file.stats.record_count) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "add_files: cannot derive the partition value of file {}: the parquet footer has no complete " + "min/max statistics for the partition source column '{}'", + file.registration_path, source_column.name); + } result.value = Null(); result.type = transform_name == "identity" || transform_name == "truncate" ? source_column.type : DataTypePtr(std::make_shared()); return result; @@ -477,9 +585,16 @@ PartitionFieldValue derivePartitionField( { if (is_date) { - /// Mirrors the native writer: identity Date partition values are the raw day - /// number promoted to UInt64 (ColumnVector -> Field). - result.value = Field(UInt64(bounds.min_int)); + /// Mirrors the native writer: identity Date partition values are the raw day number, + /// promoted to UInt64 for `Date` (ColumnVector) and Int64 for `Date32`. + if (WhichDataType(removeNullable(source_column.type)).isDate32()) + result.value = Field(Int64(bounds.min_int)); + else if (bounds.min_int >= 0 && bounds.min_int <= std::numeric_limits::max()) + result.value = Field(UInt64(bounds.min_int)); + else + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "add_files: date {} of column '{}' in file {} is out of the range of the table column type {}", + bounds.min_int, source_column.name, file.registration_path, source_column.type->getName()); } else if (is_timestamp) { @@ -626,6 +741,49 @@ std::unordered_set listLiveDataFiles( return live_paths; } +/// Whether the table metadata that `catalog` currently points to contains a snapshot with `snapshot_id`. +/// Resolves the "commit state unknown" case. Throws if the catalog or the metadata file cannot be read, +/// so that the caller never treats an unknown state as "not committed". +bool isSnapshotInCatalogMetadata( + const std::shared_ptr & catalog, + const String & table_name, + Int64 snapshot_id, + const PersistentTableComponents & persistent_components, + const DataLakeStorageSettings & data_lake_settings, + const ObjectStoragePtr & object_storage, + const ContextPtr & context, + LoggerPtr log) +{ + const auto metadata_file_info = getLatestMetadataFileAndVersionWithCatalog( + object_storage, + catalog, + table_name, + persistent_components.table_path, + data_lake_settings, + persistent_components.metadata_cache, + context, + log.get(), + persistent_components.table_uuid, + persistent_components.metadata_compression_method); + auto metadata = getMetadataJSONObject( + metadata_file_info.path, + object_storage, + persistent_components.metadata_cache, + context, + log, + metadata_file_info.compression_method, + persistent_components.table_uuid); + + if (!metadata->has(Iceberg::f_snapshots)) + return false; + for (const auto & snapshot_var : *metadata->getArray(Iceberg::f_snapshots)) + { + if (snapshot_var.extract()->getValue(Iceberg::f_metadata_snapshot_id) == snapshot_id) + return true; + } + return false; +} + Pipe resultToPipe(const AddFilesResult & result) { Block header{ @@ -680,16 +838,25 @@ AddFilesResult addFiles( persistent_components.metadata_compression_method, write_format); - auto [last_version, metadata_path, compression_method] = getLatestOrExplicitMetadataFileAndVersion( - object_storage, - persistent_components.table_path, - data_lake_settings, - persistent_components.metadata_cache, - context, - getLogger("IcebergAddFiles").get(), - persistent_components.table_uuid, - persistent_components.metadata_compression_method, - true); + /// With a catalog, the catalog names the current metadata file. Listing the storage instead may pick the + /// metadata of another table that shares the location, or miss a commit that is visible only in the catalog. + auto get_current_metadata_file = [&]() + { + return getLatestMetadataFileAndVersionWithCatalog( + object_storage, + catalog, + table_name, + persistent_components.table_path, + data_lake_settings, + persistent_components.metadata_cache, + context, + log.get(), + persistent_components.table_uuid, + persistent_components.metadata_compression_method, + /* ignore_explicit_metadata_file_path */ false); + }; + + auto [last_version, metadata_path, compression_method] = get_current_metadata_file(); filename_generator.setVersion(last_version + 1); Poco::JSON::Object::Ptr metadata = getMetadataJSONObject( @@ -762,7 +929,7 @@ AddFilesResult addFiles( std::vector analyzed_files; analyzed_files.reserve(paths.size()); for (const auto & path : paths) - analyzed_files.push_back(analyzeFile(path, resolver, object_storage, column_path_to_field_id, context, format_settings)); + analyzed_files.push_back(analyzeFile(path, resolver, object_storage, column_path_to_field_id, iceberg_type_by_field_id, context, format_settings)); /// Validate the file's columns against the table schema. if (!allow_missing_columns) @@ -826,6 +993,13 @@ AddFilesResult addFiles( throw Exception(ErrorCodes::BAD_ARGUMENTS, "add_files: file '{}' is listed twice in the input", file.registration_path); } + } + + /// Repeated on every retry: a concurrent commit may have registered the same files. + auto check_files_not_registered = [&]() + { + if (!check_duplicate_files) + return; auto live_files = listLiveDataFiles(metadata, resolver, object_storage, format_settings); for (const auto & file : analyzed_files) { @@ -833,7 +1007,10 @@ AddFilesResult addFiles( throw Exception(ErrorCodes::BAD_ARGUMENTS, "add_files: file '{}' is already registered in the current snapshot", file.registration_path); } - } + }; + check_files_not_registered(); + + const bool catalog_is_transactional = catalog && catalog->isTransactional(); /// Commit: one manifest per partition tuple, one manifest list, one append snapshot. AddFilesResult result; @@ -874,22 +1051,19 @@ AddFilesResult addFiles( auto cleanup = [&](bool retry_because_of_metadata_conflict) { + /// The new snapshot may be visible: its files must stay, even if that leaves orphans. + if (published) + { + LOG_INFO(log, "Not removing the new manifest files: the new snapshot may already be visible"); + return; + } for (size_t i = 0; i < manifest_paths.size(); ++i) object_storage->removeObjectIfExists(StoredObject(resolver.resolve(manifest_paths[i]))); object_storage->removeObjectIfExists(StoredObject(storage_manifest_list_name)); if (retry_because_of_metadata_conflict) { - auto [new_version, new_metadata_path, new_compression] = getLatestOrExplicitMetadataFileAndVersion( - object_storage, - persistent_components.table_path, - data_lake_settings, - persistent_components.metadata_cache, - context, - getLogger("IcebergAddFiles").get(), - persistent_components.table_uuid, - persistent_components.metadata_compression_method, - true); + auto [new_version, new_metadata_path, new_compression] = get_current_metadata_file(); filename_generator.setVersion(new_version + 1); metadata = getMetadataJSONObject( new_metadata_path, object_storage, persistent_components.metadata_cache, @@ -901,6 +1075,7 @@ AddFilesResult addFiles( if (metadata->getValue(Iceberg::f_default_spec_id) != partition_spec_id) throw Exception(ErrorCodes::METADATA_MISMATCH, "Table partition spec changed during add_files; restart the command"); + check_files_not_registered(); } }; @@ -988,27 +1163,78 @@ AddFilesResult addFiles( { std::string json_representation = stringifyJSON(metadata, 4); - auto hint_path = filename_generator.generateVersionHint(); - if (!writeMetadataFileAndVersionHint( - resolver, metadata_info, json_representation, hint_path, object_storage, context, - data_lake_settings[DataLakeStorageSetting::iceberg_use_version_hint])) + /// A transactional catalog writes the metadata file itself, as for native inserts. + if (!catalog_is_transactional) { - cleanup(true); - if (++retry >= max_retries) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "add_files: too many metadata conflicts, giving up"); - continue; + auto hint_path = filename_generator.generateVersionHint(); + if (!writeMetadataFileAndVersionHint( + resolver, metadata_info, json_representation, hint_path, object_storage, context, + data_lake_settings[DataLakeStorageSetting::iceberg_use_version_hint])) + { + cleanup(true); + if (++retry >= max_retries) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "add_files: too many metadata conflicts, giving up"); + continue; + } + /// The metadata file and version hint in storage now name the new snapshot. + published = true; } if (catalog) { auto catalog_filename = resolver.resolveForCatalog(metadata_info.path); const auto & [namespace_name, table] = DataLake::parseTableName(table_name); - if (!catalog->updateMetadata(namespace_name, table, catalog_filename, new_snapshot, context->getForwardedAuthToken())) + + if (!catalog_is_transactional) { - cleanup(true); - if (++retry >= max_retries) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "add_files: too many catalog conflicts, giving up"); - continue; + /// The snapshot is already visible through storage: deleting its files would break it, and a + /// retry would build on it. Keep the files and report the error. + if (!catalog->updateMetadata(namespace_name, table, catalog_filename, new_snapshot, context->getForwardedAuthToken())) + throw Exception( + ErrorCodes::DATALAKE_DATABASE_ERROR, + "add_files: Iceberg metadata {} of table {}.{} was committed to storage, but the catalog rejected " + "the update of the table pointer. The table in storage is consistent; the catalog still names " + "the previous metadata", + catalog_filename, namespace_name, table); + } + else + { + bool updated = false; + std::exception_ptr update_exception; + try + { + updated = catalog->updateMetadata(namespace_name, table, catalog_filename, new_snapshot, context->getForwardedAuthToken()); + } + catch (...) + { + update_exception = std::current_exception(); + } + + if (!updated) + { + /// Commit state unknown: the catalog may have applied the update although the client saw + /// a failure. Until the catalog proves otherwise, keep the files, even if the check throws. + published = true; + const Int64 new_snapshot_id = new_snapshot->getValue(Iceberg::f_metadata_snapshot_id); + if (isSnapshotInCatalogMetadata( + catalog, table_name, new_snapshot_id, persistent_components, data_lake_settings, + object_storage, context, log)) + { + LOG_INFO(log, "Catalog update for snapshot {} was reported as failed, but the catalog has applied it", + new_snapshot_id); + } + else + { + /// The update did not land: no snapshot references the new files. + published = false; + if (update_exception) + std::rethrow_exception(update_exception); + cleanup(true); + if (++retry >= max_retries) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "add_files: too many catalog conflicts, giving up"); + continue; + } + } } } published = true; diff --git a/tests/integration/test_storage_iceberg_with_spark/test_add_files.py b/tests/integration/test_storage_iceberg_with_spark/test_add_files.py index 8d5df5e9f1328..5bb206b24e79c 100644 --- a/tests/integration/test_storage_iceberg_with_spark/test_add_files.py +++ b/tests/integration/test_storage_iceberg_with_spark/test_add_files.py @@ -2,6 +2,8 @@ import os import tempfile +import pyarrow as pa +import pyarrow.parquet as pq import pytest from helpers.iceberg_utils import ( @@ -74,6 +76,25 @@ def write_parquet(self, subdir, filename, select_sql): data = f.read() os.remove(host_path) self.instance.exec_in_container(["rm", "-f", container_path]) + self.put_file(subdir, filename, data) + + def write_parquet_arrow(self, subdir, filename, table, **write_options): + """Write a parquet file with pyarrow, to get representations that ClickHouse itself + does not produce (other timestamp units, missing statistics, ...).""" + buffer = io.BytesIO() + pq.write_table(table, buffer, **write_options) + self.put_file(subdir, filename, buffer.getvalue()) + + def put_file(self, subdir, filename, data): + if self.storage_type == "local": + host_path = os.path.join(tempfile.gettempdir(), f"add_files_tmp_{get_uuid_str()}.parquet") + with open(host_path, "wb") as f: + f.write(data) + target_dir = f"{TABLE_PREFIX}/{self.table_name}/{subdir}" + self.instance.exec_in_container(["bash", "-c", f"mkdir -p {target_dir}"]) + self.instance.copy_file_to_container(host_path, f"{target_dir}/{filename}") + os.remove(host_path) + else: key = f"{S3_KEY_PREFIX}/{self.table_name}/{subdir}/{filename}" self.cluster.minio_client.put_object( self.cluster.minio_bucket, key, io.BytesIO(data), len(data), @@ -308,3 +329,84 @@ def test_add_files_argument_errors(started_cluster_iceberg_with_spark, storage_t env_bucket.write_parquet("data/import", "f1.parquet", "SELECT 1 AS x") error = env_bucket.add_files_error(env_bucket.table_path("data/import", "f1.parquet")) assert "bucket" in error, f"got {error}" + + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +def test_add_files_rejects_mismatched_types(started_cluster_iceberg_with_spark, storage_type): + """Footer statistics are used as Iceberg bounds as they are, so a file whose column is stored + differently from the Iceberg type must be rejected. A matching file registers, and filters + on its bounds return every matching row.""" + env = AddFilesTestEnv( + started_cluster_iceberg_with_spark, storage_type, "test_add_types", + schema="(x Nullable(Int64), d Nullable(DateTime64(6)))", + ) + + timestamps = [1704103200000000 + i * 1000000 for i in range(10)] # 2024-01-01 10:00:00 + i seconds + + # INT32 data for an Iceberg `long` column: the bounds would be 4 bytes instead of 8. + env.write_parquet_arrow("data/import", "int32.parquet", pa.table({ + "x": pa.array(range(1, 11), type=pa.int32()), + "d": pa.array(timestamps, type=pa.timestamp("us")), + })) + error = env.add_files_error(env.table_path("data/import", "int32.parquet")) + assert "does not match the Iceberg type 'long'" in error, f"got {error}" + + # Nanosecond timestamps: the bounds would be read as microseconds. + env.write_parquet_arrow("data/import", "ns.parquet", pa.table({ + "x": pa.array(range(1, 11), type=pa.int64()), + "d": pa.array([t * 1000 for t in timestamps], type=pa.timestamp("ns")), + })) + error = env.add_files_error(env.table_path("data/import", "ns.parquet")) + assert "does not match the Iceberg type" in error, f"got {error}" + + # A string column for an integer field. + env.write_parquet_arrow("data/import", "string.parquet", pa.table({ + "x": pa.array([str(i) for i in range(1, 11)], type=pa.string()), + "d": pa.array(timestamps, type=pa.timestamp("us")), + })) + error = env.add_files_error(env.table_path("data/import", "string.parquet")) + assert "does not match the Iceberg type 'long'" in error, f"got {error}" + + assert env.count() == 0 + + env.write_parquet_arrow("data/import", "ok.parquet", pa.table({ + "x": pa.array(range(1, 11), type=pa.int64()), + "d": pa.array(timestamps, type=pa.timestamp("us")), + })) + counts = env.add_files(env.table_path("data/import", "ok.parquet")) + assert counts["files_registered"] == 1, f"got {counts}" + assert env.query(f"SELECT count() FROM {env.table_name} WHERE x = 5") == "1\n" + assert env.query(f"SELECT count() FROM {env.table_name} WHERE x > 8") == "2\n" + assert env.query( + f"SELECT count() FROM {env.table_name} WHERE d >= toDateTime64('2024-01-01 10:00:08', 6, 'UTC')" + ) == "2\n" + + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +def test_add_files_partition_value_needs_statistics(started_cluster_iceberg_with_spark, storage_type): + """The partition value is derived from footer statistics. A file without them must be rejected + instead of being registered in the null partition; a file whose values are all null does belong + to the null partition.""" + env = AddFilesTestEnv( + started_cluster_iceberg_with_spark, storage_type, "test_add_no_stats", + schema="(s Nullable(String))", + partition_by="identity(s)", + ) + + values = pa.table({"s": pa.array(["abc"] * 3, type=pa.string())}) + env.write_parquet_arrow("data/import", "no_stats.parquet", values, write_statistics=False) + error = env.add_files_error(env.table_path("data/import", "no_stats.parquet")) + assert "no complete min/max statistics" in error, f"got {error}" + assert env.count() == 0 + + env.write_parquet_arrow("data/import", "stats.parquet", values) + counts = env.add_files(env.table_path("data/import", "stats.parquet")) + assert counts["partitions_touched"] == 1, f"got {counts}" + + env.write_parquet_arrow("data/import", "nulls.parquet", pa.table({"s": pa.array([None, None], type=pa.string())})) + counts = env.add_files(env.table_path("data/import", "nulls.parquet")) + assert counts["partitions_touched"] == 1, f"got {counts}" + + assert env.query(f"SELECT count() FROM {env.table_name} WHERE s = 'abc'") == "3\n" + assert env.query(f"SELECT count() FROM {env.table_name} WHERE s IS NULL") == "2\n" + assert env.count() == 5 From 480e513b9eaa7c6245b43871ced0f5107235f17c Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Wed, 18 Nov 2026 14:44:36 -0800 Subject: [PATCH 6/9] Added spark tests for iceberg add files, Added setting allow_experimental_iceberg_add_files --- src/Core/Settings.cpp | 3 + src/Core/SettingsChangesHistory.cpp | 1 + .../DataLakes/Iceberg/AddFilesExecute.cpp | 194 ++++++++++++++++-- .../DataLakes/Iceberg/IcebergMetadata.cpp | 9 + .../test_add_files.py | 165 +++++++++++++++ 5 files changed, 351 insertions(+), 21 deletions(-) diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 4150173e8bb28..342e2d1e5210f 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -9245,6 +9245,9 @@ Default age threshold in seconds for orphan file removal in Iceberg tables. File )", EXPERIMENTAL) \ DECLARE(Bool, allow_experimental_expire_snapshots, false, R"( Allow to execute experimental Iceberg command `ALTER TABLE ... EXECUTE expire_snapshots`. +)", EXPERIMENTAL) \ + DECLARE(Bool, allow_experimental_iceberg_add_files, false, R"( +Allow to execute experimental Iceberg command `ALTER TABLE ... EXECUTE add_files`, which registers existing Parquet files under the table location without rewriting them. )", EXPERIMENTAL) \ DECLARE(Bool, write_full_path_in_iceberg_metadata, false, R"( Write full paths (including s3://) into iceberg metadata files. diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 1864edcad4b3e..6742798695ef5 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -54,6 +54,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() { {"input_format_parquet_detect_variant_by_structure", false, true, "New setting to read Spark variant columns, which have no `VARIANT` logical type, as `Dynamic` instead of `Tuple`"}, {"use_iceberg_manifest_list_partition_pruning", false, true, "New setting to skip Iceberg manifest files whose manifest-list partition summaries cannot match the query filter, without reading them."}, + {"allow_experimental_iceberg_add_files", false, false, "New setting to gate the experimental Iceberg command `ALTER TABLE ... EXECUTE add_files`."}, }); addSettingsChanges(settings_changes_history, "26.8.27.20001.altinityantalya", { diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/AddFilesExecute.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/AddFilesExecute.cpp index cfb52558cf261..ea84b8a01b15a 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/AddFilesExecute.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/AddFilesExecute.cpp @@ -37,6 +37,7 @@ #include #include #include +#include #include #include @@ -66,6 +67,7 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; extern const int DATALAKE_DATABASE_ERROR; extern const int ICEBERG_SPECIFICATION_VIOLATION; +extern const int LIMIT_EXCEEDED; extern const int LOGICAL_ERROR; extern const int METADATA_MISMATCH; extern const int SUPPORT_IS_DISABLED; @@ -136,6 +138,147 @@ bool endsWithParquet(const String & path) return Poco::toLower(path).ends_with(".parquet"); } +/// The files are read and registered through the table's storage and credentials, so only files under +/// the table location may be registered. Otherwise `IcebergPathResolver::resolve` maps an outside path +/// to some key of the table's storage, which lets a user expose objects the table does not own, and the +/// manifest records a path that names a different object than the one ClickHouse read. +void checkPathUnderTableLocation(const String & path, const String & table_location) +{ + if (table_location.empty() || !path.starts_with(table_location) || path.size() <= table_location.size() + || path[table_location.size()] != '/') + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "add_files: path '{}' is not under the table location '{}'; only files under the table location can be registered", + path, table_location); + + std::string_view rest = std::string_view(path).substr(table_location.size() + 1); + while (true) + { + size_t slash = rest.find('/'); + std::string_view component = rest.substr(0, slash); + if (component == "." || component == "..") + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "add_files: path '{}' contains a '{}' component; give the path without relative components", path, component); + if (slash == std::string_view::npos) + break; + rest = rest.substr(slash + 1); + } +} + +/// The Iceberg `truncate[N]` transform of a string keeps the first N code points, as `icebergTruncate` does. +String truncateToCodePoints(const String & value, size_t code_points) +{ + size_t pos = 0; + for (size_t i = 0; i < code_points && pos < value.size(); ++i) + pos += UTF8::seqLength(static_cast(value[pos])); + return value.substr(0, std::min(pos, value.size())); +} + +/// Leaf fields (over nested structs) that must have a value in every row: the field and all its +/// ancestors are required. A file without such a column cannot be registered. +std::unordered_set collectRequiredFieldIds(Poco::JSON::Array::Ptr schema_fields) +{ + std::unordered_set result; + std::function collect; + collect = [&](Poco::JSON::Object::Ptr field) + { + if (!field->has(Iceberg::f_required) || !field->getValue(Iceberg::f_required)) + return; + auto type_var = field->get(Iceberg::f_type); + if (type_var.isString()) + { + result.insert(field->getValue(Iceberg::f_id)); + return; + } + const auto & type_obj = type_var.extract(); + if (type_obj->getValue(Iceberg::f_type) == "struct") + { + for (const auto & nested : *type_obj->getArray(Iceberg::f_fields)) + collect(nested.extract()); + } + }; + for (const auto & field_var : *schema_fields) + collect(field_var.extract()); + return result; +} + +/// Iceberg name mapping (`schema.name-mapping.default`) of a type: lets readers that resolve columns by +/// field id (Spark, Trino, iceberg-java) map the columns of files written without field ids by name. +/// Mirrors `MappingUtil.create` of iceberg-java: structs map their fields, lists map `element`, maps map +/// `key` and `value`. +Poco::JSON::Array::Ptr buildNameMappingOfType(const Poco::Dynamic::Var & type_var) +{ + if (type_var.isString()) + return nullptr; + + const auto & type_obj = type_var.extract(); + const String kind = type_obj->getValue(Iceberg::f_type); + + auto make_entry = [](Int64 id, const String & name, const Poco::JSON::Array::Ptr & nested) + { + Poco::JSON::Object::Ptr entry = new Poco::JSON::Object; + entry->set("field-id", id); + Poco::JSON::Array::Ptr names = new Poco::JSON::Array; + names->add(name); + entry->set("names", names); + if (nested) + entry->set("fields", nested); + return entry; + }; + + Poco::JSON::Array::Ptr result = new Poco::JSON::Array; + if (kind == "struct") + { + for (const auto & nested_var : *type_obj->getArray(Iceberg::f_fields)) + { + const auto & nested = nested_var.extract(); + result->add(make_entry( + nested->getValue(Iceberg::f_id), nested->getValue(Iceberg::f_name), + buildNameMappingOfType(nested->get(Iceberg::f_type)))); + } + } + else if (kind == "list") + { + result->add(make_entry( + type_obj->getValue(Iceberg::f_element_id), "element", buildNameMappingOfType(type_obj->get(Iceberg::f_element)))); + } + else if (kind == "map") + { + result->add(make_entry(type_obj->getValue(Iceberg::f_key_id), "key", buildNameMappingOfType(type_obj->get(Iceberg::f_key)))); + result->add(make_entry( + type_obj->getValue(Iceberg::f_value_id), "value", buildNameMappingOfType(type_obj->get(Iceberg::f_value)))); + } + else + { + return nullptr; + } + return result; +} + +/// Registered files are written by other tools and usually carry no Iceberg field ids. Without a name +/// mapping, readers that resolve columns by id read them as NULLs or by position. Like `add_files` of +/// iceberg-java, set the default name mapping from the current schema unless the table already has one. +void ensureDefaultNameMapping(Poco::JSON::Object::Ptr metadata, const Poco::JSON::Object::Ptr & current_schema) +{ + static constexpr auto name_mapping_property = "schema.name-mapping.default"; + + Poco::JSON::Object::Ptr properties; + if (metadata->has(Iceberg::f_properties) && !metadata->isNull(Iceberg::f_properties)) + properties = metadata->getObject(Iceberg::f_properties); + if (!properties) + { + properties = new Poco::JSON::Object; + metadata->set(Iceberg::f_properties, properties); + } + if (properties->has(name_mapping_property)) + return; + + Poco::JSON::Object::Ptr schema_as_struct = new Poco::JSON::Object; + schema_as_struct->set(Iceberg::f_type, "struct"); + schema_as_struct->set(Iceberg::f_fields, current_schema->getArray(Iceberg::f_fields)); + auto mapping = buildNameMappingOfType(Poco::Dynamic::Var(schema_as_struct)); + properties->set(name_mapping_property, stringifyJSON(mapping, /* indent */ 0)); +} + /// Expand trailing-slash paths into concrete *.parquet files below them. std::vector expandPaths( const std::vector & raw_paths, @@ -146,11 +289,13 @@ std::vector expandPaths( std::vector result; for (const auto & raw : raw_paths) { + checkPathUnderTableLocation(raw, resolver.getTableLocation()); if (!raw.ends_with('/')) { result.push_back(raw); continue; } + const size_t files_before = result.size(); RelativePathsWithMetadata children; object_storage->listObjects(resolver.resolve(IcebergPathFromMetadata::deserialize(raw)), children, /* max_keys */ 0); for (const auto & child : children) @@ -159,9 +304,14 @@ std::vector expandPaths( continue; /// Listed keys are storage paths; convert to the metadata (location-prefixed) form /// that native writes register in manifests. - result.push_back(resolver.reverseResolve(child->relative_path).serialize()); + String path = resolver.reverseResolve(child->relative_path).serialize(); + checkPathUnderTableLocation(path, resolver.getTableLocation()); + result.push_back(std::move(path)); } - LOG_DEBUG(log, "add_files: path {} expanded to {} parquet file(s)", raw, result.size()); + /// Registering nothing would commit an empty snapshot; a mistyped directory is the likely cause. + if (result.size() == files_before) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "add_files: directory '{}' contains no parquet files", raw); + LOG_DEBUG(log, "add_files: path {} expanded to {} parquet file(s)", raw, result.size() - files_before); } return result; } @@ -666,8 +816,8 @@ PartitionFieldValue derivePartitionField( { if (bounds.kind == SourceValueKind::String) { - String lo_trunc = bounds.min_str.substr(0, static_cast(transform_argument)); - String hi_trunc = bounds.max_str.substr(0, static_cast(transform_argument)); + String lo_trunc = truncateToCodePoints(bounds.min_str, static_cast(transform_argument)); + String hi_trunc = truncateToCodePoints(bounds.max_str, static_cast(transform_argument)); if (lo_trunc != hi_trunc) throw Exception(ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, "add_files: file {} spans more than one partition: truncated values of column '{}' differ", @@ -931,22 +1081,24 @@ AddFilesResult addFiles( for (const auto & path : paths) analyzed_files.push_back(analyzeFile(path, resolver, object_storage, column_path_to_field_id, iceberg_type_by_field_id, context, format_settings)); - /// Validate the file's columns against the table schema. - if (!allow_missing_columns) + /// Validate the file's columns against the table schema. A missing column reads as NULL, which a + /// required field must never return, so `allow_missing_columns` applies to optional fields only. + const auto required_field_ids = collectRequiredFieldIds(schema_fields); + for (const auto & file : analyzed_files) { - for (const auto & file : analyzed_files) + for (const auto & [field_path, field_id] : column_path_to_field_id) { - for (const auto & [field_path, field_id] : column_path_to_field_id) - { - if (iceberg_type_by_field_id.contains(field_id) && !file.bounds_by_field_id.contains(static_cast(field_id))) - { - /// Column missing from the file entirely (no statistics either way). - throw Exception(ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, - "add_files: file {} lacks column '{}' present in the table schema; " - "pass allow_missing_columns = 1 to register it anyway (values will read as the column default)", - file.registration_path, field_path); - } - } + if (!iceberg_type_by_field_id.contains(field_id) || file.bounds_by_field_id.contains(static_cast(field_id))) + continue; + if (required_field_ids.contains(field_id)) + throw Exception(ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "add_files: file {} lacks column '{}', which is required in the table schema", + file.registration_path, field_path); + if (!allow_missing_columns) + throw Exception(ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "add_files: file {} lacks column '{}' present in the table schema; " + "pass allow_missing_columns = 1 to register it anyway (its values will read as NULL)", + file.registration_path, field_path); } } @@ -1021,7 +1173,6 @@ AddFilesResult addFiles( while (true) { auto metadata_info = filename_generator.generateMetadataPathWithInfo(); - const auto storage_metadata_name = resolver.resolve(metadata_info.path); Int64 parent_snapshot = -1; if (metadata->has(Iceberg::f_current_snapshot_id) && !metadata->isNull(Iceberg::f_current_snapshot_id)) @@ -1161,6 +1312,7 @@ AddFilesResult addFiles( } { + ensureDefaultNameMapping(metadata, lookupSchema(metadata, original_schema_id)); std::string json_representation = stringifyJSON(metadata, 4); /// A transactional catalog writes the metadata file itself, as for native inserts. @@ -1173,7 +1325,7 @@ AddFilesResult addFiles( { cleanup(true); if (++retry >= max_retries) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "add_files: too many metadata conflicts, giving up"); + throw Exception(ErrorCodes::LIMIT_EXCEEDED, "add_files: too many metadata conflicts, giving up"); continue; } /// The metadata file and version hint in storage now name the new snapshot. @@ -1231,7 +1383,7 @@ AddFilesResult addFiles( std::rethrow_exception(update_exception); cleanup(true); if (++retry >= max_retries) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "add_files: too many catalog conflicts, giving up"); + throw Exception(ErrorCodes::LIMIT_EXCEEDED, "add_files: too many catalog conflicts, giving up"); continue; } } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp index 124f9d60b480f..225af0bdc9557 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp @@ -163,6 +163,7 @@ extern const SettingsBool allow_experimental_iceberg_compaction; extern const SettingsBool allow_experimental_geo_types_in_iceberg; extern const SettingsBool allow_iceberg_remove_orphan_files; extern const SettingsBool allow_experimental_expire_snapshots; +extern const SettingsBool allow_experimental_iceberg_add_files; extern const SettingsSeconds lock_acquire_timeout; extern const SettingsSeconds iceberg_compaction_delay_bias; extern const SettingsSeconds iceberg_compaction_data_cleanup; @@ -1118,6 +1119,14 @@ Pipe IcebergMetadata::executeCommand( } else if (command_name == "add_files") { + if (!context->getSettingsRef()[Setting::allow_experimental_iceberg_add_files].value) + { + throw Exception( + ErrorCodes::SUPPORT_IS_DISABLED, + "Iceberg add_files is experimental. " + "To allow its usage, enable setting allow_experimental_iceberg_add_files"); + } + checkTableRootIsQueriedPath("add_files"); return Iceberg::executeAddFiles( args, context, object_storage_, data_lake_settings, persistent_components, diff --git a/tests/integration/test_storage_iceberg_with_spark/test_add_files.py b/tests/integration/test_storage_iceberg_with_spark/test_add_files.py index 5bb206b24e79c..747813f3f6422 100644 --- a/tests/integration/test_storage_iceberg_with_spark/test_add_files.py +++ b/tests/integration/test_storage_iceberg_with_spark/test_add_files.py @@ -1,5 +1,7 @@ import io +import json import os +import re import tempfile import pyarrow as pa @@ -8,12 +10,14 @@ from helpers.iceberg_utils import ( create_iceberg_table, + default_download_directory, get_uuid_str, ) ICEBERG_SETTINGS = { "allow_insert_into_iceberg": 1, + "allow_experimental_iceberg_add_files": 1, } # Path of the table location in metadata form (with a leading slash); this is the @@ -410,3 +414,164 @@ def test_add_files_partition_value_needs_statistics(started_cluster_iceberg_with assert env.query(f"SELECT count() FROM {env.table_name} WHERE s = 'abc'") == "3\n" assert env.query(f"SELECT count() FROM {env.table_name} WHERE s IS NULL") == "2\n" assert env.count() == 5 + + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +def test_add_files_requires_setting(started_cluster_iceberg_with_spark, storage_type): + """add_files is experimental and gated by its own setting.""" + env = AddFilesTestEnv(started_cluster_iceberg_with_spark, storage_type, "test_add_setting") + env.write_parquet("data/import", "f1.parquet", "SELECT 1 AS x") + + error = env.instance.query_and_get_error( + env.add_files_query(env.table_path("data/import", "f1.parquet")), + settings={"allow_insert_into_iceberg": 1}, + ) + assert "allow_experimental_iceberg_add_files" in error, f"got {error}" + assert env.count() == 0 + + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +def test_add_files_rejects_paths_outside_table_location(started_cluster_iceberg_with_spark, storage_type): + """Files are read and registered through the table's storage, so only files under the table + location may be registered: not the files of another table, not a sibling directory sharing + the prefix, and not a path escaping the location with '..'.""" + env = AddFilesTestEnv(started_cluster_iceberg_with_spark, storage_type, "test_add_outside") + other = AddFilesTestEnv(started_cluster_iceberg_with_spark, storage_type, "test_add_outside_other") + other.write_parquet("data/import", "f1.parquet", "SELECT 1 AS x") + + for path in [ + other.table_path("data/import", "f1.parquet"), + other.table_path("data/import", ""), + f"{TABLE_PREFIX}/{env.table_name}_sibling/data/f1.parquet", + ]: + error = env.add_files_error(path) + assert "is not under the table location" in error, f"{path}: got {error}" + + error = env.add_files_error(env.table_path("data", f"../../{other.table_name}/data/import/f1.parquet")) + assert "relative components" in error, f"got {error}" + + assert env.count() == 0 + + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +def test_add_files_empty_directory(started_cluster_iceberg_with_spark, storage_type): + """A directory without parquet files is an error instead of an empty snapshot.""" + env = AddFilesTestEnv(started_cluster_iceberg_with_spark, storage_type, "test_add_empty_dir") + env.write_parquet("data/import", ".hidden.parquet", "SELECT 1 AS x") + + error = env.add_files_error(env.table_path("data/import", "")) + assert "contains no parquet files" in error, f"got {error}" + + snapshots = env.query(f"SELECT count() FROM system.iceberg_history WHERE table = '{env.table_name}'") + assert snapshots == "0\n", f"got {snapshots}" + + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +def test_add_files_missing_required_column(started_cluster_iceberg_with_spark, storage_type): + """A missing column reads as NULL, so allow_missing_columns = 1 does not cover required fields.""" + env = AddFilesTestEnv( + started_cluster_iceberg_with_spark, storage_type, "test_add_missing_required", + schema="(x Int32, y Int32)", + ) + env.write_parquet_arrow("data/import", "only_x.parquet", pa.table({"x": pa.array([1], type=pa.int32())})) + path = env.table_path("data/import", "only_x.parquet") + + error = env.add_files_error(path, allow_missing_columns=1) + assert "required in the table schema" in error, f"got {error}" + assert "'y'" in error, f"got {error}" + assert env.count() == 0 + + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +def test_add_files_rejects_unsupported_representations(started_cluster_iceberg_with_spark, storage_type): + """Decimal columns and millisecond timestamps would need their bounds converted to the Iceberg + encoding, so they are rejected rather than registered with wrong bounds.""" + env_decimal = AddFilesTestEnv( + started_cluster_iceberg_with_spark, storage_type, "test_add_decimal", + schema="(x Nullable(Decimal(9, 2)))", + ) + env_decimal.write_parquet_arrow("data/import", "decimal.parquet", pa.table({ + "x": pa.array([1, -2], type=pa.int64()).cast(pa.decimal128(9, 2)), + })) + error = env_decimal.add_files_error(env_decimal.table_path("data/import", "decimal.parquet")) + assert "which add_files does not support yet" in error, f"got {error}" + assert env_decimal.count() == 0 + + env_ts = AddFilesTestEnv( + started_cluster_iceberg_with_spark, storage_type, "test_add_ts_millis", + schema="(d Nullable(DateTime64(6)))", + partition_by="toRelativeDayNum(d)", + ) + env_ts.write_parquet_arrow("data/import", "millis.parquet", pa.table({ + "d": pa.array([1704103200000, 1704103201000], type=pa.timestamp("ms")), + }), coerce_timestamps="ms") + error = env_ts.add_files_error(env_ts.table_path("data/import", "millis.parquet")) + assert "does not match the Iceberg type" in error, f"got {error}" + assert env_ts.count() == 0 + + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +def test_add_files_truncate_multibyte(started_cluster_iceberg_with_spark, storage_type): + """truncate[N] keeps N code points, as a native insert does: a registered file lands in the same + partition as natively inserted rows, so partition pruning keeps both.""" + env = AddFilesTestEnv( + started_cluster_iceberg_with_spark, storage_type, "test_add_truncate_utf8", + schema="(s Nullable(String))", + partition_by="icebergTruncate(2, s)", + ) + env.insert("('äöü')") + + env.write_parquet_arrow("data/import", "same.parquet", pa.table({"s": pa.array(["äöü", "äöx"], type=pa.string())})) + counts = env.add_files(env.table_path("data/import", "same.parquet")) + assert counts["partitions_touched"] == 1, f"got {counts}" + + # 'äb' and 'äc' share their first 2 bytes, but not their first 2 code points. + env.write_parquet_arrow("data/import", "two.parquet", pa.table({"s": pa.array(["äb", "äc"], type=pa.string())})) + error = env.add_files_error(env.table_path("data/import", "two.parquet")) + assert "spans more than one partition" in error, f"got {error}" + + pruning = {"use_iceberg_partition_pruning": 1} + assert env.instance.query( + f"SELECT count() FROM {env.table_name} WHERE s = 'äöü'", settings=pruning + ) == "2\n" + assert env.instance.query( + f"SELECT count() FROM {env.table_name} WHERE s = 'äöx'", settings=pruning + ) == "1\n" + assert env.count() == 3 + + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +def test_add_files_spark_reads_by_name(started_cluster_iceberg_with_spark, storage_type): + """Registered files carry no Iceberg field ids. add_files sets the default name mapping, so Spark + maps their columns by name, even when the file's column order differs from the schema.""" + cluster = started_cluster_iceberg_with_spark + env = AddFilesTestEnv( + cluster, storage_type, "test_add_spark", + schema="(a Nullable(Int32), b Nullable(String))", + ) + env.write_parquet_arrow("data/import", "reordered.parquet", pa.table({ + "b": pa.array(["x", "y"], type=pa.string()), + "a": pa.array([1, 2], type=pa.int32()), + })) + env.add_files(env.table_path("data/import", "reordered.parquet")) + assert env.query(f"SELECT a, b FROM {env.table_name} ORDER BY a") == "1\tx\n2\ty\n" + + local_dir = f"/var/lib/clickhouse/user_files/iceberg_data/default/{env.table_name}/" + default_download_directory(cluster, storage_type, local_dir, local_dir) + metadata_dir = os.path.join(local_dir, "metadata") + versions = {} + for name in os.listdir(metadata_dir): + match = re.match(r"v(\d+)(?:[-.].*)?\.metadata\.json$", name) + if match: + versions[int(match.group(1))] = name + latest = max(versions) + with open(os.path.join(metadata_dir, "version-hint.text"), "w") as hint: + hint.write(str(latest)) + + with open(os.path.join(metadata_dir, versions[latest])) as f: + properties = json.load(f).get("properties", {}) + mapping = json.loads(properties["schema.name-mapping.default"]) + assert sorted(entry["names"][0] for entry in mapping) == ["a", "b"], f"got {mapping}" + + rows = cluster.spark_session.read.format("iceberg").load(local_dir).collect() + assert sorted((row["a"], row["b"]) for row in rows) == [(1, "x"), (2, "y")] From 8c6e269129b69da3d43b258aa0945e4d65c1b016 Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Thu, 19 Nov 2026 07:32:17 -0800 Subject: [PATCH 7/9] Address issue with name mapping going stale after schema changes --- src/Databases/DataLake/ICatalog.cpp | 14 ++ src/Databases/DataLake/ICatalog.h | 13 ++ src/Databases/DataLake/RestCatalog.cpp | 30 ++++- src/Databases/DataLake/RestCatalog.h | 13 +- src/Databases/DataLake/UnityV2Catalog.cpp | 15 +++ src/Databases/DataLake/UnityV2Catalog.h | 8 ++ .../gtest_rest_catalog_update_metadata.cpp | 22 +++ .../DataLakes/Iceberg/AddFilesExecute.cpp | 111 +++++++++++++-- .../test_add_files.py | 127 +++++++++++++----- 9 files changed, 304 insertions(+), 49 deletions(-) diff --git a/src/Databases/DataLake/ICatalog.cpp b/src/Databases/DataLake/ICatalog.cpp index 9cc28afc48917..afa6feceba484 100644 --- a/src/Databases/DataLake/ICatalog.cpp +++ b/src/Databases/DataLake/ICatalog.cpp @@ -523,6 +523,20 @@ bool ICatalog::updateMetadata(const String & /*namespace_name*/, const String & throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "updateMetadata is not implemented"); } +bool ICatalog::updateMetadataAndProperties( + const String & namespace_name, + const String & table_name, + const String & new_metadata_path, + Poco::JSON::Object::Ptr new_snapshot, + const std::map & properties_to_set, + const DB::ForwardedAuthTokenPtr & auth_token) const +{ + if (!properties_to_set.empty() && isTransactional()) + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, + "Setting table properties together with a snapshot is not implemented for this catalog"); + return updateMetadata(namespace_name, table_name, new_metadata_path, new_snapshot, auth_token); +} + bool ICatalog::updateSchema( const String & /*namespace_name*/, const String & /*table_name*/, diff --git a/src/Databases/DataLake/ICatalog.h b/src/Databases/DataLake/ICatalog.h index 5e092e92b75f2..0fd70ad24de41 100644 --- a/src/Databases/DataLake/ICatalog.h +++ b/src/Databases/DataLake/ICatalog.h @@ -15,6 +15,7 @@ #include #include +#include #include #include @@ -333,6 +334,18 @@ class ICatalog /// Updates metadata in catalog. virtual bool updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot, const DB::ForwardedAuthTokenPtr & auth_token) const; + /// Like `updateMetadata`, and also sets the table properties `properties_to_set` in the same commit. + /// A non-transactional catalog only stores a pointer to `new_metadata_path`, which already holds the + /// properties, so the default implementation delegates to `updateMetadata` for it. A transactional + /// catalog applies the updates itself and must override this to send the properties. + virtual bool updateMetadataAndProperties( + const String & namespace_name, + const String & table_name, + const String & new_metadata_path, + Poco::JSON::Object::Ptr new_snapshot, + const std::map & properties_to_set, + const DB::ForwardedAuthTokenPtr & auth_token) const; + /// Commit a schema evolution (ADD/DROP/MODIFY/RENAME COLUMN) to the catalog. /// `new_metadata_path` is the path of the freshly written `vN.metadata.json`; it is used by /// non-transactional catalogs (e.g. Glue) that only store a pointer to the metadata file. diff --git a/src/Databases/DataLake/RestCatalog.cpp b/src/Databases/DataLake/RestCatalog.cpp index 6160a8bb57ed2..8c7a6d0d1f529 100644 --- a/src/Databases/DataLake/RestCatalog.cpp +++ b/src/Databases/DataLake/RestCatalog.cpp @@ -429,7 +429,10 @@ Poco::JSON::Object::Ptr buildUpdateSchemaRequestBody( } Poco::JSON::Object::Ptr buildUpdateMetadataRequestBody( - const String & namespace_name, const String & table_name, Poco::JSON::Object::Ptr new_snapshot) + const String & namespace_name, + const String & table_name, + Poco::JSON::Object::Ptr new_snapshot, + const std::map & properties_to_set) { if (!new_snapshot) return nullptr; @@ -475,6 +478,16 @@ Poco::JSON::Object::Ptr buildUpdateMetadataRequestBody( set_snapshot->set("snapshot-id", new_snapshot->getValue("snapshot-id")); updates->add(set_snapshot); } + if (!properties_to_set.empty()) + { + Poco::JSON::Object::Ptr set_properties = new Poco::JSON::Object; + set_properties->set("action", "set-properties"); + Poco::JSON::Object::Ptr properties = new Poco::JSON::Object; + for (const auto & [key, value] : properties_to_set) + properties->set(key, value); + set_properties->set("updates", properties); + updates->add(set_properties); + } request_body->set("updates", updates); return request_body; @@ -2751,7 +2764,18 @@ bool RestCatalog::createTable( } -bool RestCatalog::updateMetadata(const String & namespace_name, const String & table_name, const String & /*new_metadata_path*/, Poco::JSON::Object::Ptr new_snapshot, const DB::ForwardedAuthTokenPtr & auth_token) const +bool RestCatalog::updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot, const DB::ForwardedAuthTokenPtr & auth_token) const +{ + return updateMetadataAndProperties(namespace_name, table_name, new_metadata_path, new_snapshot, /* properties_to_set */ {}, auth_token); +} + +bool RestCatalog::updateMetadataAndProperties( + const String & namespace_name, + const String & table_name, + const String & /*new_metadata_path*/, + Poco::JSON::Object::Ptr new_snapshot, + const std::map & properties_to_set, + const DB::ForwardedAuthTokenPtr & auth_token) const { loadConfigIfNeeded(auth_token); @@ -2764,7 +2788,7 @@ bool RestCatalog::updateMetadata(const String & namespace_name, const String & t const auto state_snapshot = getStateSnapshot(); const std::string endpoint = (base_url / state_snapshot->config.prefix / NAMESPACES_ENDPOINT / encodeNamespaceForURI(namespace_name) / "tables" / table_name).generic_string(); - auto request_body = buildUpdateMetadataRequestBody(namespace_name, table_name, new_snapshot); + auto request_body = buildUpdateMetadataRequestBody(namespace_name, table_name, new_snapshot, properties_to_set); try { diff --git a/src/Databases/DataLake/RestCatalog.h b/src/Databases/DataLake/RestCatalog.h index f088e0c6f658f..1d3bccf5ddf17 100644 --- a/src/Databases/DataLake/RestCatalog.h +++ b/src/Databases/DataLake/RestCatalog.h @@ -121,6 +121,14 @@ class RestCatalog : public ICatalog, public DB::WithContext bool updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot, const DB::ForwardedAuthTokenPtr & auth_token) const override; + bool updateMetadataAndProperties( + const String & namespace_name, + const String & table_name, + const String & new_metadata_path, + Poco::JSON::Object::Ptr new_snapshot, + const std::map & properties_to_set, + const DB::ForwardedAuthTokenPtr & auth_token) const override; + bool updateSchema( const String & namespace_name, const String & table_name, @@ -609,10 +617,13 @@ Poco::JSON::Object::Ptr buildUpdateSchemaRequestBody( Int32 previous_schema_id, Int32 new_last_column_id); +/// Builds the JSON body that commits `new_snapshot` (and sets `properties_to_set` in the same commit) +/// via the Iceberg REST catalog. Poco::JSON::Object::Ptr buildUpdateMetadataRequestBody( const String & namespace_name, const String & table_name, - Poco::JSON::Object::Ptr new_snapshot); + Poco::JSON::Object::Ptr new_snapshot, + const std::map & properties_to_set = {}); } diff --git a/src/Databases/DataLake/UnityV2Catalog.cpp b/src/Databases/DataLake/UnityV2Catalog.cpp index 14a497d1d3735..33facbb7ef837 100644 --- a/src/Databases/DataLake/UnityV2Catalog.cpp +++ b/src/Databases/DataLake/UnityV2Catalog.cpp @@ -439,6 +439,21 @@ bool UnityV2Catalog::updateMetadata( }); } +bool UnityV2Catalog::updateMetadataAndProperties( + const String & namespace_name, + const String & table_name, + const String & new_metadata_path, + Poco::JSON::Object::Ptr new_snapshot, + const std::map & properties_to_set, + const DB::ForwardedAuthTokenPtr & auth_token) const +{ + return requestWithRetry([&](bool force_refresh) + { + return getIcebergRestCatalog(force_refresh)->updateMetadataAndProperties( + namespace_name, table_name, new_metadata_path, new_snapshot, properties_to_set, auth_token); + }); +} + bool UnityV2Catalog::updateSchema( const String & namespace_name, const String & table_name, diff --git a/src/Databases/DataLake/UnityV2Catalog.h b/src/Databases/DataLake/UnityV2Catalog.h index d40a90f125c22..672741699a65c 100644 --- a/src/Databases/DataLake/UnityV2Catalog.h +++ b/src/Databases/DataLake/UnityV2Catalog.h @@ -91,6 +91,14 @@ class UnityV2Catalog final : public ICatalog, private DB::WithContext Poco::JSON::Object::Ptr new_snapshot, const DB::ForwardedAuthTokenPtr & auth_token) const override; + bool updateMetadataAndProperties( + const String & namespace_name, + const String & table_name, + const String & new_metadata_path, + Poco::JSON::Object::Ptr new_snapshot, + const std::map & properties_to_set, + const DB::ForwardedAuthTokenPtr & auth_token) const override; + bool updateSchema( const String & namespace_name, const String & table_name, diff --git a/src/Databases/DataLake/tests/gtest_rest_catalog_update_metadata.cpp b/src/Databases/DataLake/tests/gtest_rest_catalog_update_metadata.cpp index 8b209262cf628..50541508dd6c5 100644 --- a/src/Databases/DataLake/tests/gtest_rest_catalog_update_metadata.cpp +++ b/src/Databases/DataLake/tests/gtest_rest_catalog_update_metadata.cpp @@ -131,6 +131,28 @@ TEST(RestCatalogUpdateMetadataBody, SnapshotUpdateParentMinusOneNoRequirement) EXPECT_FALSE(req->has("snapshot-id")); } +TEST(RestCatalogUpdateMetadataBody, PropertiesAreSetInTheSameCommit) +{ + Poco::JSON::Object::Ptr snapshot = new Poco::JSON::Object; + snapshot->set("snapshot-id", static_cast(7)); + + auto without_properties = DataLake::buildUpdateMetadataRequestBody("ns", "t", snapshot); + ASSERT_TRUE(without_properties); + EXPECT_FALSE(findUpdateByAction(without_properties->getArray("updates"), "set-properties")); + + auto body = DataLake::buildUpdateMetadataRequestBody( + "ns", "t", snapshot, {{"schema.name-mapping.default", R"([{"field-id":1,"names":["a"]}])"}}); + ASSERT_TRUE(body); + auto updates = body->getArray("updates"); + ASSERT_TRUE(findUpdateByAction(updates, "add-snapshot")); + ASSERT_TRUE(findUpdateByAction(updates, "set-snapshot-ref")); + auto set_properties = findUpdateByAction(updates, "set-properties"); + ASSERT_TRUE(set_properties); + EXPECT_EQ( + set_properties->getObject("updates")->getValue("schema.name-mapping.default"), + R"([{"field-id":1,"names":["a"]}])"); +} + TEST(RestCatalogUpdateSchemaBody, EquivalentSchemaDeduplicates) { Poco::JSON::Object::Ptr metadata = new Poco::JSON::Object; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/AddFilesExecute.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/AddFilesExecute.cpp index ea84b8a01b15a..f3cbb5115eba1 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/AddFilesExecute.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/AddFilesExecute.cpp @@ -41,6 +41,7 @@ #include #include +#include #include #include @@ -56,6 +57,7 @@ #include #include #include +#include #include #include @@ -254,13 +256,58 @@ Poco::JSON::Array::Ptr buildNameMappingOfType(const Poco::Dynamic::Var & type_va return result; } +constexpr auto name_mapping_property = "schema.name-mapping.default"; + +/// Names that an existing name mapping gives each field id, over all nesting levels. +void collectMappedNames(const Poco::JSON::Array::Ptr & mapping, std::unordered_map> & names_by_field_id) +{ + for (const auto & entry_var : *mapping) + { + const auto & entry = entry_var.extract(); + if (entry->has("field-id") && entry->has("names")) + { + auto & names = names_by_field_id[entry->getValue("field-id")]; + for (const auto & name_var : *entry->getArray("names")) + names.push_back(name_var.convert()); + } + if (entry->has("fields")) + collectMappedNames(entry->getArray("fields"), names_by_field_id); + } +} + +/// Keep the names that the existing mapping gives each field before its own current name: files written +/// before a rename still carry the old name. +void mergeMappedNames(const Poco::JSON::Array::Ptr & mapping, const std::unordered_map> & old_names_by_field_id) +{ + for (const auto & entry_var : *mapping) + { + /// A copy of the pointer: the entry itself is modified. + Poco::JSON::Object::Ptr entry = entry_var.extract(); + auto old_it = old_names_by_field_id.find(entry->getValue("field-id")); + if (old_it != old_names_by_field_id.end()) + { + const String current_name = entry->getArray("names")->getElement(0); + Poco::JSON::Array::Ptr names = new Poco::JSON::Array; + std::unordered_set seen; + for (const auto & name : old_it->second) + if (seen.insert(name).second) + names->add(name); + if (seen.insert(current_name).second) + names->add(current_name); + entry->set("names", names); + } + if (entry->has("fields")) + mergeMappedNames(entry->getArray("fields"), old_names_by_field_id); + } +} + /// Registered files are written by other tools and usually carry no Iceberg field ids. Without a name /// mapping, readers that resolve columns by id read them as NULLs or by position. Like `add_files` of -/// iceberg-java, set the default name mapping from the current schema unless the table already has one. -void ensureDefaultNameMapping(Poco::JSON::Object::Ptr metadata, const Poco::JSON::Object::Ptr & current_schema) +/// iceberg-java, set the default name mapping from the current schema. An existing mapping is extended +/// rather than kept as is: a field added since it was written would otherwise read as NULL from the +/// registered files. Returns the new value of the property if it changed; `metadata` is updated too. +std::optional updateDefaultNameMapping(Poco::JSON::Object::Ptr metadata, const Poco::JSON::Object::Ptr & current_schema) { - static constexpr auto name_mapping_property = "schema.name-mapping.default"; - Poco::JSON::Object::Ptr properties; if (metadata->has(Iceberg::f_properties) && !metadata->isNull(Iceberg::f_properties)) properties = metadata->getObject(Iceberg::f_properties); @@ -269,14 +316,28 @@ void ensureDefaultNameMapping(Poco::JSON::Object::Ptr metadata, const Poco::JSON properties = new Poco::JSON::Object; metadata->set(Iceberg::f_properties, properties); } - if (properties->has(name_mapping_property)) - return; Poco::JSON::Object::Ptr schema_as_struct = new Poco::JSON::Object; schema_as_struct->set(Iceberg::f_type, "struct"); schema_as_struct->set(Iceberg::f_fields, current_schema->getArray(Iceberg::f_fields)); auto mapping = buildNameMappingOfType(Poco::Dynamic::Var(schema_as_struct)); - properties->set(name_mapping_property, stringifyJSON(mapping, /* indent */ 0)); + + std::optional old_value; + if (properties->has(name_mapping_property)) + { + old_value = properties->getValue(name_mapping_property); + Poco::JSON::Parser parser; + auto old_mapping = parser.parse(*old_value).extract(); + std::unordered_map> old_names_by_field_id; + collectMappedNames(old_mapping, old_names_by_field_id); + mergeMappedNames(mapping, old_names_by_field_id); + } + + String new_value = stringifyJSON(mapping, /* indent */ 0); + if (old_value == new_value) + return std::nullopt; + properties->set(name_mapping_property, new_value); + return new_value; } /// Expand trailing-slash paths into concrete *.parquet files below them. @@ -381,6 +442,8 @@ struct AnalyzedFile std::unordered_map bounds_by_field_id; /// Field ids whose null count is known for every row group of the file. std::unordered_map complete_null_counts; + /// Field ids whose parquet column cannot hold nulls (it and all its parents are `REQUIRED`). + std::unordered_set non_nullable_field_ids; }; /// Check that the parquet column has exactly the representation that the Iceberg type requires, @@ -552,6 +615,8 @@ AnalyzedFile analyzeFile( continue; checkParquetColumnType(*descriptor, type_it->second, registration_path); columns_with_bounds.insert(static_cast(field_it->second)); + if (descriptor->max_definition_level() == 0) + result.non_nullable_field_ids.insert(static_cast(field_it->second)); } std::unordered_map column_sizes; @@ -1088,8 +1153,27 @@ AddFilesResult addFiles( { for (const auto & [field_path, field_id] : column_path_to_field_id) { - if (!iceberg_type_by_field_id.contains(field_id) || file.bounds_by_field_id.contains(static_cast(field_id))) + if (!iceberg_type_by_field_id.contains(field_id)) + continue; + if (file.bounds_by_field_id.contains(static_cast(field_id))) + { + /// A required field must have a value in every row. Readers return the type default or fail + /// for nulls in it, so the file must prove that the column has none. + if (required_field_ids.contains(field_id) && !file.non_nullable_field_ids.contains(static_cast(field_id))) + { + auto null_count_it = file.complete_null_counts.find(static_cast(field_id)); + if (null_count_it == file.complete_null_counts.end()) + throw Exception(ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "add_files: column '{}' of file {} is required in the table schema, but the file may contain " + "nulls in it: the column is optional in the file and the parquet footer has no complete null counts", + field_path, file.registration_path); + if (null_count_it->second != 0) + throw Exception(ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "add_files: column '{}' of file {} is required in the table schema, but the file contains {} null(s) in it", + field_path, file.registration_path, null_count_it->second); + } continue; + } if (required_field_ids.contains(field_id)) throw Exception(ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, "add_files: file {} lacks column '{}', which is required in the table schema", @@ -1312,7 +1396,10 @@ AddFilesResult addFiles( } { - ensureDefaultNameMapping(metadata, lookupSchema(metadata, original_schema_id)); + /// A transactional catalog builds the new metadata itself, so the changed property is sent with the commit. + std::map properties_to_set; + if (auto name_mapping = updateDefaultNameMapping(metadata, lookupSchema(metadata, original_schema_id))) + properties_to_set.emplace(name_mapping_property, *name_mapping); std::string json_representation = stringifyJSON(metadata, 4); /// A transactional catalog writes the metadata file itself, as for native inserts. @@ -1341,7 +1428,8 @@ AddFilesResult addFiles( { /// The snapshot is already visible through storage: deleting its files would break it, and a /// retry would build on it. Keep the files and report the error. - if (!catalog->updateMetadata(namespace_name, table, catalog_filename, new_snapshot, context->getForwardedAuthToken())) + if (!catalog->updateMetadataAndProperties( + namespace_name, table, catalog_filename, new_snapshot, properties_to_set, context->getForwardedAuthToken())) throw Exception( ErrorCodes::DATALAKE_DATABASE_ERROR, "add_files: Iceberg metadata {} of table {}.{} was committed to storage, but the catalog rejected " @@ -1355,7 +1443,8 @@ AddFilesResult addFiles( std::exception_ptr update_exception; try { - updated = catalog->updateMetadata(namespace_name, table, catalog_filename, new_snapshot, context->getForwardedAuthToken()); + updated = catalog->updateMetadataAndProperties( + namespace_name, table, catalog_filename, new_snapshot, properties_to_set, context->getForwardedAuthToken()); } catch (...) { diff --git a/tests/integration/test_storage_iceberg_with_spark/test_add_files.py b/tests/integration/test_storage_iceberg_with_spark/test_add_files.py index 747813f3f6422..e2400a5d8705f 100644 --- a/tests/integration/test_storage_iceberg_with_spark/test_add_files.py +++ b/tests/integration/test_storage_iceberg_with_spark/test_add_files.py @@ -158,7 +158,7 @@ def test_add_files_basic(started_cluster_iceberg_with_spark, storage_type, forma """Register one external parquet file into an empty table and read it back.""" env = AddFilesTestEnv(started_cluster_iceberg_with_spark, storage_type, "test_add_basic", format_version=format_version) - env.write_parquet("data/import", "f1.parquet", "SELECT number + 1 AS x FROM numbers(3)") + env.write_parquet("data/import", "f1.parquet", "SELECT toInt32(number + 1) AS x FROM numbers(3)") counts = env.add_files(env.table_path("data/import", "f1.parquet")) assert counts["files_registered"] == 1, f"got {counts}" @@ -173,8 +173,8 @@ def test_add_files_array_form_multiple(started_cluster_iceberg_with_spark, stora """The array form registers several files in one snapshot.""" env = AddFilesTestEnv(started_cluster_iceberg_with_spark, storage_type, "test_add_array") - env.write_parquet("data/import", "f1.parquet", "SELECT number AS x FROM numbers(3)") - env.write_parquet("data/import", "f2.parquet", "SELECT number + 10 AS x FROM numbers(3)") + env.write_parquet("data/import", "f1.parquet", "SELECT toInt32(number) AS x FROM numbers(3)") + env.write_parquet("data/import", "f2.parquet", "SELECT toInt32(number + 10) AS x FROM numbers(3)") counts = env.add_files([ env.table_path("data/import", "f1.parquet"), env.table_path("data/import", "f2.parquet"), @@ -192,7 +192,7 @@ def test_add_files_onto_nonempty_table(started_cluster_iceberg_with_spark, stora env = AddFilesTestEnv(started_cluster_iceberg_with_spark, storage_type, "test_add_nonempty") env.insert("(1), (2)") - env.write_parquet("data/import", "f1.parquet", "SELECT number + 100 AS x FROM numbers(3)") + env.write_parquet("data/import", "f1.parquet", "SELECT toInt32(number + 100) AS x FROM numbers(3)") counts = env.add_files(env.table_path("data/import", "f1.parquet")) assert counts["files_registered"] == 1, f"got {counts}" @@ -205,9 +205,9 @@ def test_add_files_trailing_slash_expansion(started_cluster_iceberg_with_spark, """A path ending with '/' registers every *.parquet below it, skipping hidden files.""" env = AddFilesTestEnv(started_cluster_iceberg_with_spark, storage_type, "test_add_expand") - env.write_parquet("data/import", "f1.parquet", "SELECT 1 AS x") - env.write_parquet("data/import", "f2.parquet", "SELECT 2 AS x") - env.write_parquet("data/import", ".hidden.parquet", "SELECT 3 AS x") + env.write_parquet("data/import", "f1.parquet", "SELECT toInt32(1) AS x") + env.write_parquet("data/import", "f2.parquet", "SELECT toInt32(2) AS x") + env.write_parquet("data/import", ".hidden.parquet", "SELECT toInt32(3) AS x") counts = env.add_files(env.table_path("data/import", "")) @@ -225,17 +225,17 @@ def test_add_files_partitioned_identity(started_cluster_iceberg_with_spark, stor partition_by="identity(x)", ) - env.write_parquet("data/import", "f7.parquet", "SELECT 7 AS x FROM numbers(3)") + env.write_parquet("data/import", "f7.parquet", "SELECT toInt32(7) AS x FROM numbers(3)") counts = env.add_files(env.table_path("data/import", "f7.parquet")) assert counts["files_registered"] == 1, f"got {counts}" assert counts["partitions_touched"] == 1, f"got {counts}" - env.write_parquet("data/import", "f8.parquet", "SELECT 8 AS x FROM numbers(2)") + env.write_parquet("data/import", "f8.parquet", "SELECT toInt32(8) AS x FROM numbers(2)") counts = env.add_files(env.table_path("data/import", "f8.parquet")) assert counts["partitions_touched"] == 1, f"got {counts}" assert env.query(f"SELECT * FROM {env.table_name} ORDER BY x") == "7\n7\n7\n8\n8\n" - env.write_parquet("data/import", "mixed.parquet", "SELECT 7 AS x UNION ALL SELECT 8 AS x") + env.write_parquet("data/import", "mixed.parquet", "SELECT toInt32(7) AS x UNION ALL SELECT toInt32(8) AS x") error = env.add_files_error(env.table_path("data/import", "mixed.parquet")) assert "spans more than one partition" in error, f"got {error}" # The failed registration must not have committed anything. @@ -277,7 +277,7 @@ def test_add_files_duplicate_detection(started_cluster_iceberg_with_spark, stora default; check_duplicate_files = 0 disables the check.""" env = AddFilesTestEnv(started_cluster_iceberg_with_spark, storage_type, "test_add_dup") - env.write_parquet("data/import", "f1.parquet", "SELECT 1 AS x") + env.write_parquet("data/import", "f1.parquet", "SELECT toInt32(1) AS x") path = env.table_path("data/import", "f1.parquet") counts = env.add_files(path) @@ -303,7 +303,7 @@ def test_add_files_allow_missing_columns(started_cluster_iceberg_with_spark, sto schema="(x Int32, y Nullable(String))", ) - env.write_parquet("data/import", "only_x.parquet", "SELECT 1 AS x") + env.write_parquet("data/import", "only_x.parquet", "SELECT toInt32(1) AS x") path = env.table_path("data/import", "only_x.parquet") error = env.add_files_error(path) @@ -330,9 +330,9 @@ def test_add_files_argument_errors(started_cluster_iceberg_with_spark, storage_t started_cluster_iceberg_with_spark, storage_type, "test_add_bucket", partition_by="icebergBucket(4, x)", ) - env_bucket.write_parquet("data/import", "f1.parquet", "SELECT 1 AS x") + env_bucket.write_parquet("data/import", "f1.parquet", "SELECT toInt32(1) AS x") error = env_bucket.add_files_error(env_bucket.table_path("data/import", "f1.parquet")) - assert "bucket" in error, f"got {error}" + assert "bucket transform" in error, f"got {error}" @pytest.mark.parametrize("storage_type", ["local", "s3"]) @@ -420,7 +420,7 @@ def test_add_files_partition_value_needs_statistics(started_cluster_iceberg_with def test_add_files_requires_setting(started_cluster_iceberg_with_spark, storage_type): """add_files is experimental and gated by its own setting.""" env = AddFilesTestEnv(started_cluster_iceberg_with_spark, storage_type, "test_add_setting") - env.write_parquet("data/import", "f1.parquet", "SELECT 1 AS x") + env.write_parquet("data/import", "f1.parquet", "SELECT toInt32(1) AS x") error = env.instance.query_and_get_error( env.add_files_query(env.table_path("data/import", "f1.parquet")), @@ -437,7 +437,7 @@ def test_add_files_rejects_paths_outside_table_location(started_cluster_iceberg_ the prefix, and not a path escaping the location with '..'.""" env = AddFilesTestEnv(started_cluster_iceberg_with_spark, storage_type, "test_add_outside") other = AddFilesTestEnv(started_cluster_iceberg_with_spark, storage_type, "test_add_outside_other") - other.write_parquet("data/import", "f1.parquet", "SELECT 1 AS x") + other.write_parquet("data/import", "f1.parquet", "SELECT toInt32(1) AS x") for path in [ other.table_path("data/import", "f1.parquet"), @@ -457,7 +457,7 @@ def test_add_files_rejects_paths_outside_table_location(started_cluster_iceberg_ def test_add_files_empty_directory(started_cluster_iceberg_with_spark, storage_type): """A directory without parquet files is an error instead of an empty snapshot.""" env = AddFilesTestEnv(started_cluster_iceberg_with_spark, storage_type, "test_add_empty_dir") - env.write_parquet("data/import", ".hidden.parquet", "SELECT 1 AS x") + env.write_parquet("data/import", ".hidden.parquet", "SELECT toInt32(1) AS x") error = env.add_files_error(env.table_path("data/import", "")) assert "contains no parquet files" in error, f"got {error}" @@ -540,10 +540,42 @@ def test_add_files_truncate_multibyte(started_cluster_iceberg_with_spark, storag assert env.count() == 3 +def download_for_spark(cluster, storage_type, table_name): + """Copy the table to the host for Spark and point the version hint at the latest metadata file. + Returns the local table directory and the parsed latest metadata.""" + local_dir = f"/var/lib/clickhouse/user_files/iceberg_data/default/{table_name}/" + default_download_directory(cluster, storage_type, local_dir, local_dir) + metadata_dir = os.path.join(local_dir, "metadata") + versions = {} + for name in os.listdir(metadata_dir): + match = re.match(r"v(\d+)(?:[-.].*)?\.metadata\.json$", name) + if match: + versions[int(match.group(1))] = name + latest = max(versions) + with open(os.path.join(metadata_dir, "version-hint.text"), "w") as hint: + hint.write(str(latest)) + with open(os.path.join(metadata_dir, versions[latest])) as f: + return local_dir, json.load(f) + + +def name_mapping_of(metadata): + """Field id -> names of the default name mapping, over all nesting levels.""" + result = {} + + def walk(entries): + for entry in entries: + result[entry["field-id"]] = entry["names"] + walk(entry.get("fields", [])) + + walk(json.loads(metadata["properties"]["schema.name-mapping.default"])) + return result + + @pytest.mark.parametrize("storage_type", ["local", "s3"]) def test_add_files_spark_reads_by_name(started_cluster_iceberg_with_spark, storage_type): """Registered files carry no Iceberg field ids. add_files sets the default name mapping, so Spark - maps their columns by name, even when the file's column order differs from the schema.""" + maps their columns by name, even when the file's column order differs from the schema. A column + added later is added to the existing mapping, so Spark reads it from files registered after that.""" cluster = started_cluster_iceberg_with_spark env = AddFilesTestEnv( cluster, storage_type, "test_add_spark", @@ -556,22 +588,49 @@ def test_add_files_spark_reads_by_name(started_cluster_iceberg_with_spark, stora env.add_files(env.table_path("data/import", "reordered.parquet")) assert env.query(f"SELECT a, b FROM {env.table_name} ORDER BY a") == "1\tx\n2\ty\n" - local_dir = f"/var/lib/clickhouse/user_files/iceberg_data/default/{env.table_name}/" - default_download_directory(cluster, storage_type, local_dir, local_dir) - metadata_dir = os.path.join(local_dir, "metadata") - versions = {} - for name in os.listdir(metadata_dir): - match = re.match(r"v(\d+)(?:[-.].*)?\.metadata\.json$", name) - if match: - versions[int(match.group(1))] = name - latest = max(versions) - with open(os.path.join(metadata_dir, "version-hint.text"), "w") as hint: - hint.write(str(latest)) + local_dir, metadata = download_for_spark(cluster, storage_type, env.table_name) + assert sorted(names[0] for names in name_mapping_of(metadata).values()) == ["a", "b"] + rows = cluster.spark_session.read.format("iceberg").load(local_dir).collect() + assert sorted((row["a"], row["b"]) for row in rows) == [(1, "x"), (2, "y")] - with open(os.path.join(metadata_dir, versions[latest])) as f: - properties = json.load(f).get("properties", {}) - mapping = json.loads(properties["schema.name-mapping.default"]) - assert sorted(entry["names"][0] for entry in mapping) == ["a", "b"], f"got {mapping}" + env.query(f"ALTER TABLE {env.table_name} ADD COLUMN c Nullable(String)") + env.write_parquet_arrow("data/import", "with_c.parquet", pa.table({ + "c": pa.array(["p"], type=pa.string()), + "a": pa.array([3], type=pa.int32()), + "b": pa.array(["z"], type=pa.string()), + })) + env.add_files(env.table_path("data/import", "with_c.parquet")) + assert env.query(f"SELECT a, b, c FROM {env.table_name} ORDER BY a") == "1\tx\t\\N\n2\ty\t\\N\n3\tz\tp\n" + local_dir, metadata = download_for_spark(cluster, storage_type, env.table_name) + assert sorted(names[0] for names in name_mapping_of(metadata).values()) == ["a", "b", "c"] rows = cluster.spark_session.read.format("iceberg").load(local_dir).collect() - assert sorted((row["a"], row["b"]) for row in rows) == [(1, "x"), (2, "y")] + assert sorted((row["a"], row["b"], row["c"]) for row in rows) == [(1, "x", None), (2, "y", None), (3, "z", "p")] + + +@pytest.mark.parametrize("storage_type", ["local", "s3"]) +def test_add_files_nulls_in_required_column(started_cluster_iceberg_with_spark, storage_type): + """A required field must have a value in every row, so a file with nulls in its column, or without + the null counts that prove it has none, is rejected. A column that cannot hold nulls needs no counts.""" + env = AddFilesTestEnv( + started_cluster_iceberg_with_spark, storage_type, "test_add_required_nulls", + schema="(x Int32)", + ) + + env.write_parquet_arrow("data/import", "nulls.parquet", pa.table({"x": pa.array([1, None], type=pa.int32())})) + error = env.add_files_error(env.table_path("data/import", "nulls.parquet")) + assert "contains 1 null(s)" in error, f"got {error}" + + env.write_parquet_arrow( + "data/import", "no_stats.parquet", pa.table({"x": pa.array([1, 2], type=pa.int32())}), write_statistics=False) + error = env.add_files_error(env.table_path("data/import", "no_stats.parquet")) + assert "may contain nulls" in error, f"got {error}" + assert env.count() == 0 + + required = pa.schema([pa.field("x", pa.int32(), nullable=False)]) + env.write_parquet_arrow( + "data/import", "required.parquet", pa.table({"x": pa.array([1, 2], type=pa.int32())}, schema=required), + write_statistics=False) + counts = env.add_files(env.table_path("data/import", "required.parquet")) + assert counts["files_registered"] == 1, f"got {counts}" + assert env.count() == 2 From 6fe26ebed31428da44b46c3b2e1c4d9c073a6d94 Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Thu, 19 Nov 2026 07:52:06 -0800 Subject: [PATCH 8/9] Rolled back change with mapping to be in sync with iceberg-java --- .../DataLakes/Iceberg/AddFilesExecute.cpp | 74 +++---------------- .../test_add_files.py | 17 +---- 2 files changed, 10 insertions(+), 81 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/AddFilesExecute.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/AddFilesExecute.cpp index f3cbb5115eba1..a56f7e716a494 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/AddFilesExecute.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/AddFilesExecute.cpp @@ -41,7 +41,6 @@ #include #include -#include #include #include @@ -258,54 +257,12 @@ Poco::JSON::Array::Ptr buildNameMappingOfType(const Poco::Dynamic::Var & type_va constexpr auto name_mapping_property = "schema.name-mapping.default"; -/// Names that an existing name mapping gives each field id, over all nesting levels. -void collectMappedNames(const Poco::JSON::Array::Ptr & mapping, std::unordered_map> & names_by_field_id) -{ - for (const auto & entry_var : *mapping) - { - const auto & entry = entry_var.extract(); - if (entry->has("field-id") && entry->has("names")) - { - auto & names = names_by_field_id[entry->getValue("field-id")]; - for (const auto & name_var : *entry->getArray("names")) - names.push_back(name_var.convert()); - } - if (entry->has("fields")) - collectMappedNames(entry->getArray("fields"), names_by_field_id); - } -} - -/// Keep the names that the existing mapping gives each field before its own current name: files written -/// before a rename still carry the old name. -void mergeMappedNames(const Poco::JSON::Array::Ptr & mapping, const std::unordered_map> & old_names_by_field_id) -{ - for (const auto & entry_var : *mapping) - { - /// A copy of the pointer: the entry itself is modified. - Poco::JSON::Object::Ptr entry = entry_var.extract(); - auto old_it = old_names_by_field_id.find(entry->getValue("field-id")); - if (old_it != old_names_by_field_id.end()) - { - const String current_name = entry->getArray("names")->getElement(0); - Poco::JSON::Array::Ptr names = new Poco::JSON::Array; - std::unordered_set seen; - for (const auto & name : old_it->second) - if (seen.insert(name).second) - names->add(name); - if (seen.insert(current_name).second) - names->add(current_name); - entry->set("names", names); - } - if (entry->has("fields")) - mergeMappedNames(entry->getArray("fields"), old_names_by_field_id); - } -} - /// Registered files are written by other tools and usually carry no Iceberg field ids. Without a name /// mapping, readers that resolve columns by id read them as NULLs or by position. Like `add_files` of -/// iceberg-java, set the default name mapping from the current schema. An existing mapping is extended -/// rather than kept as is: a field added since it was written would otherwise read as NULL from the -/// registered files. Returns the new value of the property if it changed; `metadata` is updated too. +/// iceberg-java, set the default name mapping from the current schema unless the table already has one. +/// An existing mapping is kept as is: keeping it current with schema changes is the job of the schema +/// update (as `UpdateSchema` does in iceberg-java), not of `add_files`. +/// Returns the value of the property if it was set; `metadata` is updated too. std::optional updateDefaultNameMapping(Poco::JSON::Object::Ptr metadata, const Poco::JSON::Object::Ptr & current_schema) { Poco::JSON::Object::Ptr properties; @@ -316,28 +273,15 @@ std::optional updateDefaultNameMapping(Poco::JSON::Object::Ptr metadata, properties = new Poco::JSON::Object; metadata->set(Iceberg::f_properties, properties); } + if (properties->has(name_mapping_property)) + return std::nullopt; Poco::JSON::Object::Ptr schema_as_struct = new Poco::JSON::Object; schema_as_struct->set(Iceberg::f_type, "struct"); schema_as_struct->set(Iceberg::f_fields, current_schema->getArray(Iceberg::f_fields)); - auto mapping = buildNameMappingOfType(Poco::Dynamic::Var(schema_as_struct)); - - std::optional old_value; - if (properties->has(name_mapping_property)) - { - old_value = properties->getValue(name_mapping_property); - Poco::JSON::Parser parser; - auto old_mapping = parser.parse(*old_value).extract(); - std::unordered_map> old_names_by_field_id; - collectMappedNames(old_mapping, old_names_by_field_id); - mergeMappedNames(mapping, old_names_by_field_id); - } - - String new_value = stringifyJSON(mapping, /* indent */ 0); - if (old_value == new_value) - return std::nullopt; - properties->set(name_mapping_property, new_value); - return new_value; + String value = stringifyJSON(buildNameMappingOfType(Poco::Dynamic::Var(schema_as_struct)), /* indent */ 0); + properties->set(name_mapping_property, value); + return value; } /// Expand trailing-slash paths into concrete *.parquet files below them. diff --git a/tests/integration/test_storage_iceberg_with_spark/test_add_files.py b/tests/integration/test_storage_iceberg_with_spark/test_add_files.py index e2400a5d8705f..7d00ba5e3eddf 100644 --- a/tests/integration/test_storage_iceberg_with_spark/test_add_files.py +++ b/tests/integration/test_storage_iceberg_with_spark/test_add_files.py @@ -574,8 +574,7 @@ def walk(entries): @pytest.mark.parametrize("storage_type", ["local", "s3"]) def test_add_files_spark_reads_by_name(started_cluster_iceberg_with_spark, storage_type): """Registered files carry no Iceberg field ids. add_files sets the default name mapping, so Spark - maps their columns by name, even when the file's column order differs from the schema. A column - added later is added to the existing mapping, so Spark reads it from files registered after that.""" + maps their columns by name, even when the file's column order differs from the schema.""" cluster = started_cluster_iceberg_with_spark env = AddFilesTestEnv( cluster, storage_type, "test_add_spark", @@ -593,20 +592,6 @@ def test_add_files_spark_reads_by_name(started_cluster_iceberg_with_spark, stora rows = cluster.spark_session.read.format("iceberg").load(local_dir).collect() assert sorted((row["a"], row["b"]) for row in rows) == [(1, "x"), (2, "y")] - env.query(f"ALTER TABLE {env.table_name} ADD COLUMN c Nullable(String)") - env.write_parquet_arrow("data/import", "with_c.parquet", pa.table({ - "c": pa.array(["p"], type=pa.string()), - "a": pa.array([3], type=pa.int32()), - "b": pa.array(["z"], type=pa.string()), - })) - env.add_files(env.table_path("data/import", "with_c.parquet")) - assert env.query(f"SELECT a, b, c FROM {env.table_name} ORDER BY a") == "1\tx\t\\N\n2\ty\t\\N\n3\tz\tp\n" - - local_dir, metadata = download_for_spark(cluster, storage_type, env.table_name) - assert sorted(names[0] for names in name_mapping_of(metadata).values()) == ["a", "b", "c"] - rows = cluster.spark_session.read.format("iceberg").load(local_dir).collect() - assert sorted((row["a"], row["b"], row["c"]) for row in rows) == [(1, "x", None), (2, "y", None), (3, "z", "p")] - @pytest.mark.parametrize("storage_type", ["local", "s3"]) def test_add_files_nulls_in_required_column(started_cluster_iceberg_with_spark, storage_type): From 45013626851f14bb5e0e5459554d7e77ec247f26 Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Thu, 19 Nov 2026 10:39:56 -0800 Subject: [PATCH 9/9] Fix failing tess --- .../test_storage_iceberg_with_spark/test_add_files.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_storage_iceberg_with_spark/test_add_files.py b/tests/integration/test_storage_iceberg_with_spark/test_add_files.py index 7d00ba5e3eddf..f48fce9012327 100644 --- a/tests/integration/test_storage_iceberg_with_spark/test_add_files.py +++ b/tests/integration/test_storage_iceberg_with_spark/test_add_files.py @@ -3,6 +3,7 @@ import os import re import tempfile +from decimal import Decimal import pyarrow as pa import pyarrow.parquet as pq @@ -491,7 +492,7 @@ def test_add_files_rejects_unsupported_representations(started_cluster_iceberg_w schema="(x Nullable(Decimal(9, 2)))", ) env_decimal.write_parquet_arrow("data/import", "decimal.parquet", pa.table({ - "x": pa.array([1, -2], type=pa.int64()).cast(pa.decimal128(9, 2)), + "x": pa.array([Decimal("1.00"), Decimal("-2.00")], type=pa.decimal128(9, 2)), })) error = env_decimal.add_files_error(env_decimal.table_path("data/import", "decimal.parquet")) assert "which add_files does not support yet" in error, f"got {error}"