Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/Core/Settings.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -9290,6 +9290,9 @@ Default age threshold in seconds for orphan file removal in Iceberg tables. File
)", EXPERIMENTAL) \
DECLARE(Bool, allow_experimental_expire_snapshots, false, R"(
Allow to execute experimental Iceberg command `ALTER TABLE ... EXECUTE expire_snapshots`.
)", EXPERIMENTAL) \
DECLARE(Bool, allow_experimental_iceberg_add_files, false, R"(
Allow to execute experimental Iceberg command `ALTER TABLE ... EXECUTE add_files`, which registers existing Parquet files under the table location without rewriting them.
)", EXPERIMENTAL) \
DECLARE(Bool, write_full_path_in_iceberg_metadata, false, R"(
Write full paths (including s3://) into iceberg metadata files.
Expand Down
1 change: 1 addition & 0 deletions src/Core/SettingsChangesHistory.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory()
{
{"input_format_parquet_detect_variant_by_structure", false, true, "New setting to read Spark variant columns, which have no `VARIANT` logical type, as `Dynamic` instead of `Tuple`"},
{"use_iceberg_manifest_list_partition_pruning", false, true, "New setting to skip Iceberg manifest files whose manifest-list partition summaries cannot match the query filter, without reading them."},
{"allow_experimental_iceberg_add_files", false, false, "New setting to gate the experimental Iceberg command `ALTER TABLE ... EXECUTE add_files`."},
});
addSettingsChanges(settings_changes_history, "26.8.27.20001.altinityantalya",
{
Expand Down
14 changes: 14 additions & 0 deletions src/Databases/DataLake/ICatalog.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -523,6 +523,20 @@ bool ICatalog::updateMetadata(const String & /*namespace_name*/, const String &
throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "updateMetadata is not implemented");
}

bool ICatalog::updateMetadataAndProperties(
const String & namespace_name,
const String & table_name,
const String & new_metadata_path,
Poco::JSON::Object::Ptr new_snapshot,
const std::map<String, String> & properties_to_set,
const DB::ForwardedAuthTokenPtr & auth_token) const
{
if (!properties_to_set.empty() && isTransactional())
throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED,
"Setting table properties together with a snapshot is not implemented for this catalog");
return updateMetadata(namespace_name, table_name, new_metadata_path, new_snapshot, auth_token);
}

bool ICatalog::updateSchema(
const String & /*namespace_name*/,
const String & /*table_name*/,
Expand Down
13 changes: 13 additions & 0 deletions src/Databases/DataLake/ICatalog.h
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
#include <Poco/JSON/Object.h>

#include <functional>
#include <map>
#include <string_view>
#include <unordered_map>

Expand Down Expand Up @@ -333,6 +334,18 @@ class ICatalog
/// Updates metadata in catalog.
virtual bool updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot, const DB::ForwardedAuthTokenPtr & auth_token) const;

/// Like `updateMetadata`, and also sets the table properties `properties_to_set` in the same commit.
/// A non-transactional catalog only stores a pointer to `new_metadata_path`, which already holds the
/// properties, so the default implementation delegates to `updateMetadata` for it. A transactional
/// catalog applies the updates itself and must override this to send the properties.
virtual bool updateMetadataAndProperties(
const String & namespace_name,
const String & table_name,
const String & new_metadata_path,
Poco::JSON::Object::Ptr new_snapshot,
const std::map<String, String> & properties_to_set,
const DB::ForwardedAuthTokenPtr & auth_token) const;

/// Commit a schema evolution (ADD/DROP/MODIFY/RENAME COLUMN) to the catalog.
/// `new_metadata_path` is the path of the freshly written `vN.metadata.json`; it is used by
/// non-transactional catalogs (e.g. Glue) that only store a pointer to the metadata file.
Expand Down
30 changes: 27 additions & 3 deletions src/Databases/DataLake/RestCatalog.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -429,7 +429,10 @@ Poco::JSON::Object::Ptr buildUpdateSchemaRequestBody(
}

Poco::JSON::Object::Ptr buildUpdateMetadataRequestBody(
const String & namespace_name, const String & table_name, Poco::JSON::Object::Ptr new_snapshot)
const String & namespace_name,
const String & table_name,
Poco::JSON::Object::Ptr new_snapshot,
const std::map<String, String> & properties_to_set)
{
if (!new_snapshot)
return nullptr;
Expand Down Expand Up @@ -475,6 +478,16 @@ Poco::JSON::Object::Ptr buildUpdateMetadataRequestBody(
set_snapshot->set("snapshot-id", new_snapshot->getValue<Int64>("snapshot-id"));
updates->add(set_snapshot);
}
if (!properties_to_set.empty())
{
Poco::JSON::Object::Ptr set_properties = new Poco::JSON::Object;
set_properties->set("action", "set-properties");
Poco::JSON::Object::Ptr properties = new Poco::JSON::Object;
for (const auto & [key, value] : properties_to_set)
properties->set(key, value);
set_properties->set("updates", properties);
updates->add(set_properties);
}
request_body->set("updates", updates);

return request_body;
Expand Down Expand Up @@ -2751,7 +2764,18 @@ bool RestCatalog::createTable(
}


bool RestCatalog::updateMetadata(const String & namespace_name, const String & table_name, const String & /*new_metadata_path*/, Poco::JSON::Object::Ptr new_snapshot, const DB::ForwardedAuthTokenPtr & auth_token) const
bool RestCatalog::updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot, const DB::ForwardedAuthTokenPtr & auth_token) const
{
return updateMetadataAndProperties(namespace_name, table_name, new_metadata_path, new_snapshot, /* properties_to_set */ {}, auth_token);
}

bool RestCatalog::updateMetadataAndProperties(
const String & namespace_name,
const String & table_name,
const String & /*new_metadata_path*/,
Poco::JSON::Object::Ptr new_snapshot,
const std::map<String, String> & properties_to_set,
const DB::ForwardedAuthTokenPtr & auth_token) const
{
loadConfigIfNeeded(auth_token);

Expand All @@ -2764,7 +2788,7 @@ bool RestCatalog::updateMetadata(const String & namespace_name, const String & t
const auto state_snapshot = getStateSnapshot();
const std::string endpoint = (base_url / state_snapshot->config.prefix / NAMESPACES_ENDPOINT / encodeNamespaceForURI(namespace_name) / "tables" / table_name).generic_string();

auto request_body = buildUpdateMetadataRequestBody(namespace_name, table_name, new_snapshot);
auto request_body = buildUpdateMetadataRequestBody(namespace_name, table_name, new_snapshot, properties_to_set);

try
{
Expand Down
13 changes: 12 additions & 1 deletion src/Databases/DataLake/RestCatalog.h
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,14 @@ class RestCatalog : public ICatalog, public DB::WithContext

bool updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot, const DB::ForwardedAuthTokenPtr & auth_token) const override;

bool updateMetadataAndProperties(
const String & namespace_name,
const String & table_name,
const String & new_metadata_path,
Poco::JSON::Object::Ptr new_snapshot,
const std::map<String, String> & properties_to_set,
const DB::ForwardedAuthTokenPtr & auth_token) const override;

bool updateSchema(
const String & namespace_name,
const String & table_name,
Expand Down Expand Up @@ -609,10 +617,13 @@ Poco::JSON::Object::Ptr buildUpdateSchemaRequestBody(
Int32 previous_schema_id,
Int32 new_last_column_id);

/// Builds the JSON body that commits `new_snapshot` (and sets `properties_to_set` in the same commit)
/// via the Iceberg REST catalog.
Poco::JSON::Object::Ptr buildUpdateMetadataRequestBody(
const String & namespace_name,
const String & table_name,
Poco::JSON::Object::Ptr new_snapshot);
Poco::JSON::Object::Ptr new_snapshot,
const std::map<String, String> & properties_to_set = {});

}

Expand Down
15 changes: 15 additions & 0 deletions src/Databases/DataLake/UnityV2Catalog.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -439,6 +439,21 @@ bool UnityV2Catalog::updateMetadata(
});
}

bool UnityV2Catalog::updateMetadataAndProperties(
const String & namespace_name,
const String & table_name,
const String & new_metadata_path,
Poco::JSON::Object::Ptr new_snapshot,
const std::map<String, String> & properties_to_set,
const DB::ForwardedAuthTokenPtr & auth_token) const
{
return requestWithRetry([&](bool force_refresh)
{
return getIcebergRestCatalog(force_refresh)->updateMetadataAndProperties(
namespace_name, table_name, new_metadata_path, new_snapshot, properties_to_set, auth_token);
});
}

bool UnityV2Catalog::updateSchema(
const String & namespace_name,
const String & table_name,
Expand Down
8 changes: 8 additions & 0 deletions src/Databases/DataLake/UnityV2Catalog.h
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,14 @@ class UnityV2Catalog final : public ICatalog, private DB::WithContext
Poco::JSON::Object::Ptr new_snapshot,
const DB::ForwardedAuthTokenPtr & auth_token) const override;

bool updateMetadataAndProperties(
const String & namespace_name,
const String & table_name,
const String & new_metadata_path,
Poco::JSON::Object::Ptr new_snapshot,
const std::map<String, String> & properties_to_set,
const DB::ForwardedAuthTokenPtr & auth_token) const override;

bool updateSchema(
const String & namespace_name,
const String & table_name,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,28 @@ TEST(RestCatalogUpdateMetadataBody, SnapshotUpdateParentMinusOneNoRequirement)
EXPECT_FALSE(req->has("snapshot-id"));
}

TEST(RestCatalogUpdateMetadataBody, PropertiesAreSetInTheSameCommit)
{
Poco::JSON::Object::Ptr snapshot = new Poco::JSON::Object;
snapshot->set("snapshot-id", static_cast<Int64>(7));

auto without_properties = DataLake::buildUpdateMetadataRequestBody("ns", "t", snapshot);
ASSERT_TRUE(without_properties);
EXPECT_FALSE(findUpdateByAction(without_properties->getArray("updates"), "set-properties"));

auto body = DataLake::buildUpdateMetadataRequestBody(
"ns", "t", snapshot, {{"schema.name-mapping.default", R"([{"field-id":1,"names":["a"]}])"}});
ASSERT_TRUE(body);
auto updates = body->getArray("updates");
ASSERT_TRUE(findUpdateByAction(updates, "add-snapshot"));
ASSERT_TRUE(findUpdateByAction(updates, "set-snapshot-ref"));
auto set_properties = findUpdateByAction(updates, "set-properties");
ASSERT_TRUE(set_properties);
EXPECT_EQ(
set_properties->getObject("updates")->getValue<std::string>("schema.name-mapping.default"),
R"([{"field-id":1,"names":["a"]}])");
}

TEST(RestCatalogUpdateSchemaBody, EquivalentSchemaDeduplicates)
{
Poco::JSON::Object::Ptr metadata = new Poco::JSON::Object;
Expand Down
Loading
Loading