Skip to content
Open
54 changes: 48 additions & 6 deletions src/Storages/IStorageCluster.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
#include <pcg_random.hpp>
#include <Common/randomSeed.h>

#include <Analyzer/createUniqueAliasesIfNecessary.h>
#include <Functions/FunctionsLogical.h>
#include <Functions/IFunctionAdaptors.h>

Expand Down Expand Up @@ -441,7 +442,8 @@ or (object_storage_cluster_join_mode='global')
void IStorageCluster::updateQueryWithJoinToSendIfNeeded(
ASTPtr & query_to_send,
SelectQueryInfo query_info,
const ContextPtr & context)
const ContextPtr & context,
QueryTreeNodePtr * resolved_query_tree)
{
auto object_storage_cluster_join_mode = context->getSettingsRef()[Setting::object_storage_cluster_join_mode];
switch (object_storage_cluster_join_mode)
Expand All @@ -453,7 +455,7 @@ void IStorageCluster::updateQueryWithJoinToSendIfNeeded(
if (!needsInitiatorLocalJoin(info))
return;

rewriteQueryForInitiatorLocalJoin(query_to_send, query_info, info, context);
rewriteQueryForInitiatorLocalJoin(query_to_send, query_info, info, context, resolved_query_tree);
return;
}
case ObjectStorageClusterJoinMode::GLOBAL:
Expand All @@ -464,7 +466,7 @@ void IStorageCluster::updateQueryWithJoinToSendIfNeeded(
if (queryHasRightOrFullJoin(query_to_send))
{
auto info = getQueryJoinInfoFromAST(query_to_send);
rewriteQueryForInitiatorLocalJoin(query_to_send, query_info, info, context);
rewriteQueryForInitiatorLocalJoin(query_to_send, query_info, info, context, resolved_query_tree);
}
return;
}
Expand Down Expand Up @@ -723,14 +725,50 @@ void IStorageCluster::read(
storage_snapshot->check(column_names);

/// Calculate the header. This is significant, because some columns could be thrown away in some cases like query with count(*)
///
/// The header must match what the shard returns for `query_to_send`. The shard re-analyzes that AST and
/// numbers `__tableN` from 1, so both cases below renumber a clone of the resolved tree and avoid a second
/// catalog resolution:
/// 1. A join stays on the initiator and `query_to_send` was rewritten: use the rewritten left-scan tree.
/// 2. `query_to_send` is unchanged: use the original resolved tree. `FetchColumns` plans it directly,
/// and higher stages use `getHeaderForProcessingStage`.

SharedHeader sample_block;

updateQueryWithJoinToSendIfNeeded(query_to_send, query_info, context);
/// A join kept on the initiator replaces `query_to_send` but already has a resolved left-scan tree.
QueryTreeNodePtr resolved_query_tree;
updateQueryWithJoinToSendIfNeeded(query_to_send, query_info, context, &resolved_query_tree);

if (settings[Setting::allow_experimental_analyzer])
{
sample_block = InterpreterSelectQueryAnalyzer::getSampleBlock(query_to_send, context, SelectQueryOptions(processed_stage));
/// Re-analyzing the shard AST resolves a `DataLakeCatalog` table again and repeats a remote catalog lookup.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: I would make this comment a bit more descriptive, something along the following lines:

Try to avoid unnecessary catalog calls in the following cases:

  1. query tree has already been resolved, this is ok because of xyz
  2. ...

I still don`t understand case 2 very well tho, looking into it

if (canReuseResolvedQueryTreeForHeader() && resolved_query_tree)
{
/// The rewritten left scan keeps the outer alias (`__table2` when this table is not leftmost).
/// A shard re-analyzes the sent AST and `createUniqueAliasesIfNecessary` restarts at `__table1`.
auto header_tree = resolved_query_tree->clone();
createUniqueAliasesIfNecessary(header_tree, context);
sample_block = InterpreterSelectQueryAnalyzer::getSampleBlock(
header_tree, context, SelectQueryOptions(processed_stage));
}
else if (canReuseResolvedQueryTreeForHeader() && query_to_send == query_info.query && query_info.query_tree)
{
/// A shard re-analyzes this query and `createUniqueAliasesIfNecessary` restarts `__tableN` at 1.
/// The resolved tree still has the outer alias (`__table2` when this table is not leftmost).
auto query_info_for_header = query_info;
query_info_for_header.query_tree = query_info.query_tree->clone();
createUniqueAliasesIfNecessary(query_info_for_header.query_tree, context);
/// A `SECONDARY_QUERY` is planned only to `FetchColumns`. `getHeaderForProcessingStage` then
/// returns snapshot names (`symbol`), while the shard interpreter returns `__table1.symbol`.
/// Planning the resolved tree keeps those identifiers and does not resolve the catalog again.
if (processed_stage == QueryProcessingStage::FetchColumns)
sample_block = InterpreterSelectQueryAnalyzer::getSampleBlock(
query_info_for_header.query_tree, context, SelectQueryOptions(processed_stage));
else
sample_block = getHeaderForProcessingStage(column_names, storage_snapshot, query_info_for_header, context, processed_stage);
}
else
sample_block = InterpreterSelectQueryAnalyzer::getSampleBlock(query_to_send, context, SelectQueryOptions(processed_stage));
}
else
{
Expand Down Expand Up @@ -1060,7 +1098,8 @@ void IStorageCluster::rewriteQueryForInitiatorLocalJoin(
ASTPtr & query_to_send,
const SelectQueryInfo & query_info,
const QueryTreeInfo & info,
const ContextPtr & context)
const ContextPtr & context,
QueryTreeNodePtr * resolved_query_tree)
{
/// Analyzer path: reuse `extractLeftTableExpression` and `buildQueryToReadColumnsFromTableExpression`
/// (same helpers the planner uses when wrapping `IStorageCluster` in a subquery).
Expand Down Expand Up @@ -1102,6 +1141,9 @@ void IStorageCluster::rewriteQueryForInitiatorLocalJoin(
rewritten_query_node.getPrewhere() = query_node.getPrewhere();
rewritten_query_node.getWhere() = query_node.getWhere();

if (resolved_query_tree)
*resolved_query_tree = rewritten_query_tree;

query_to_send = queryNodeToDistributedSelectQuery(rewritten_query_tree);
return;
}
Expand Down
11 changes: 9 additions & 2 deletions src/Storages/IStorageCluster.h
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,13 @@ class IStorageCluster : public IStorage

protected:
virtual void updateBeforeRead(const ContextPtr &) {}
void updateQueryWithJoinToSendIfNeeded(ASTPtr & query_to_send, SelectQueryInfo query_info, const ContextPtr & context);
void updateQueryWithJoinToSendIfNeeded(
ASTPtr & query_to_send,
SelectQueryInfo query_info,
const ContextPtr & context,
QueryTreeNodePtr * resolved_query_tree = nullptr);
/// `DataLakeCatalog` tables need this to avoid resolving the catalog table a second time for the header.
virtual bool canReuseResolvedQueryTreeForHeader() const { return false; }

virtual void updateConfigurationIfNeeded(ContextPtr /* context */) {}

Expand Down Expand Up @@ -184,7 +190,8 @@ class IStorageCluster : public IStorage
ASTPtr & query_to_send,
const SelectQueryInfo & query_info,
const QueryTreeInfo & info,
const ContextPtr & context);
const ContextPtr & context,
QueryTreeNodePtr * resolved_query_tree);
};


Expand Down
1 change: 1 addition & 0 deletions src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ StorageObjectStorageCluster::StorageObjectStorageCluster(
cluster_name_, table_id_, getLogger(fmt::format("{}({})", configuration_->getEngineName(), table_id_.table_name)))
, configuration{configuration_}
, object_storage(object_storage_)
, is_datalake_catalog_table(is_datalake_query)
{
configuration->initPartitionStrategy(partition_by, columns_in_table_or_function_definition, context_);
configuration->check(context_);
Expand Down
6 changes: 6 additions & 0 deletions src/Storages/ObjectStorage/StorageObjectStorageCluster.h
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,10 @@ class StorageObjectStorageCluster : public IStorageCluster

/// Whether this query is served by `pure_storage` instead of being distributed over a cluster.
bool readsFromPureStorage(ContextPtr context) const;
/// Only `DataLakeCatalog` tables may reuse the resolved query tree for the header. Re-analyzing their
/// AST resolves the table through the REST catalog again. Other cluster reads keep re-analyzing the AST,
/// so their headers do not change.
bool canReuseResolvedQueryTreeForHeader() const override { return is_datalake_catalog_table; }

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why only catalogs can do it?


void readFallBackToPure(
QueryPlan & query_plan,
Expand Down Expand Up @@ -243,6 +247,8 @@ class StorageObjectStorageCluster : public IStorageCluster
const ObjectStoragePtr object_storage;
/// True for explicit *Cluster(...); false for SETTINGS / ENGINE / DataLake paths.
bool cluster_name_from_function_argument = false;
/// True only for tables resolved through a `DataLakeCatalog` database.
bool is_datalake_catalog_table = false;

/// non-clustered storage to fall back on pure realisation if needed
std::shared_ptr<StorageObjectStorage> pure_storage;
Expand Down
195 changes: 195 additions & 0 deletions tests/integration/test_database_iceberg/test.py
Original file line number Diff line number Diff line change
Expand Up @@ -1441,6 +1441,201 @@ def test_cluster_select(started_cluster):
assert node2.query(f"SELECT * FROM {CATALOG_NAME}.`{root_namespace}.{table_name}`", settings={"parallel_replicas_for_cluster_engines": 1, "enable_parallel_replicas": 2, "cluster_for_parallel_replicas": "cluster_simple"}) == 'pablo\n'


def test_object_storage_cluster_loads_rest_catalog_table_once(started_cluster):

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🤖 [Posted by an LLM agent on behalf of @mkmkme]

These three tests only check DataLakeRestCatalogGetTableMetadata = 1. If the header from the reused tree didn't match what the shard sends, they would only catch it for SELECT * and a simple join. Could you also check the results of an aggregate (SELECT count() ... WHERE ..., as in #2721) and of a UNION ALL of two catalog tables? Those are the shapes that broke in swarms.

node = started_cluster.instances["node1"]

test_ref = f"test_object_storage_cluster_{uuid.uuid4().hex}"
table_name = f"{test_ref}_table"
namespace = f"{test_ref}_namespace"

catalog = load_catalog_impl(started_cluster)
catalog.create_namespace(namespace)
table = create_table(catalog, namespace, table_name)
table.append(pa.Table.from_pylist([generate_record()]))

create_clickhouse_iceberg_database(started_cluster, node, CATALOG_NAME)

query_id = f"object_storage_cluster_catalog_{uuid.uuid4().hex}"
assert (
node.query(
f"SELECT * FROM {CATALOG_NAME}.`{namespace}.{table_name}` "
"SETTINGS object_storage_cluster = 'cluster_simple' FORMAT Null",
query_id=query_id,
)
== ""
)

node.query("SYSTEM FLUSH LOGS system.query_log")
assert (
node.query(
"SELECT ProfileEvents['DataLakeRestCatalogGetTableMetadata'] "
"FROM system.query_log "
f"WHERE query_id = '{query_id}' AND type = 'QueryFinish' AND is_initial_query"
)
== "1\n"
)


def test_object_storage_cluster_join_loads_rest_catalog_table_once(started_cluster):
node = started_cluster.instances["node1"]

test_ref = f"test_object_storage_cluster_join_{uuid.uuid4().hex}"
table_name = f"{test_ref}_table"
namespace = f"{test_ref}_namespace"
memory_table = f"{test_ref}_memory"

catalog = load_catalog_impl(started_cluster)
catalog.create_namespace(namespace)
table = create_table(catalog, namespace, table_name)
table.append(pa.Table.from_pylist([generate_record()]))

create_clickhouse_iceberg_database(started_cluster, node, CATALOG_NAME)
node.query(f"CREATE TABLE {memory_table} (symbol String) ENGINE = Memory")
node.query(f"INSERT INTO {memory_table} VALUES ('kek')")

query_id = f"object_storage_cluster_join_catalog_{uuid.uuid4().hex}"
assert (
node.query(
f"SELECT t.symbol FROM {CATALOG_NAME}.`{namespace}.{table_name}` AS t "
f"INNER JOIN {memory_table} AS m ON t.symbol = m.symbol "
"SETTINGS object_storage_cluster = 'cluster_simple'",
query_id=query_id,
)
== "kek\n"
)

node.query("SYSTEM FLUSH LOGS system.query_log")
assert (
node.query(
"SELECT ProfileEvents['DataLakeRestCatalogGetTableMetadata'] "
"FROM system.query_log "
f"WHERE query_id = '{query_id}' AND type = 'QueryFinish' AND is_initial_query"
)
== "1\n"
)


def test_object_storage_cluster_right_join_loads_rest_catalog_table_once(started_cluster):
node = started_cluster.instances["node1"]

test_ref = f"test_object_storage_cluster_right_join_{uuid.uuid4().hex}"
table_name = f"{test_ref}_table"
namespace = f"{test_ref}_namespace"
memory_table = f"{test_ref}_memory"

catalog = load_catalog_impl(started_cluster)
catalog.create_namespace(namespace)
table = create_table(catalog, namespace, table_name)
table.append(pa.Table.from_pylist([generate_record()]))

create_clickhouse_iceberg_database(started_cluster, node, CATALOG_NAME)
node.query(f"CREATE TABLE {memory_table} (symbol String) ENGINE = Memory")
node.query(f"INSERT INTO {memory_table} VALUES ('kek')")

query_id = f"object_storage_cluster_right_join_catalog_{uuid.uuid4().hex}"
assert (
node.query(
f"SELECT t.symbol FROM {memory_table} AS m "
f"INNER JOIN {CATALOG_NAME}.`{namespace}.{table_name}` AS t ON m.symbol = t.symbol "
"SETTINGS object_storage_cluster = 'cluster_simple'",
query_id=query_id,
)
== "kek\n"
)

node.query("SYSTEM FLUSH LOGS system.query_log")
assert (
node.query(
"SELECT ProfileEvents['DataLakeRestCatalogGetTableMetadata'] "
"FROM system.query_log "
f"WHERE query_id = '{query_id}' AND type = 'QueryFinish' AND is_initial_query"
)
== "1\n"
)


def test_object_storage_cluster_right_in_subquery_loads_rest_catalog_table_once(started_cluster):
node = started_cluster.instances["node1"]

test_ref = f"test_object_storage_cluster_right_in_{uuid.uuid4().hex}"
table_name = f"{test_ref}_table"
namespace = f"{test_ref}_namespace"
memory_table = f"{test_ref}_memory"

catalog = load_catalog_impl(started_cluster)
catalog.create_namespace(namespace)
table = create_table(catalog, namespace, table_name)
table.append(pa.Table.from_pylist([generate_record()]))

create_clickhouse_iceberg_database(started_cluster, node, CATALOG_NAME)
node.query(f"CREATE TABLE {memory_table} (symbol String) ENGINE = Memory")
node.query(f"INSERT INTO {memory_table} VALUES ('kek')")

query_id = f"object_storage_cluster_right_in_catalog_{uuid.uuid4().hex}"
assert (
node.query(
f"SELECT t.symbol FROM {memory_table} AS m "
f"INNER JOIN {CATALOG_NAME}.`{namespace}.{table_name}` AS t ON m.symbol = t.symbol "
"WHERE t.symbol IN (SELECT 'kek' FROM system.one) "
"SETTINGS object_storage_cluster = 'cluster_simple'",
query_id=query_id,
)
== "kek\n"
)

node.query("SYSTEM FLUSH LOGS system.query_log")
assert (
node.query(
"SELECT ProfileEvents['DataLakeRestCatalogGetTableMetadata'] "
"FROM system.query_log "
f"WHERE query_id = '{query_id}' AND type = 'QueryFinish' AND is_initial_query"
)
== "1\n"
)


def test_object_storage_cluster_secondary_query_returns_shard_identifiers(started_cluster):
node1 = started_cluster.instances["node1"]
node2 = started_cluster.instances["node2"]

test_ref = f"test_object_storage_cluster_secondary_{uuid.uuid4().hex}"
table_name = f"{test_ref}_table"
namespace = f"{test_ref}_namespace"

catalog = load_catalog_impl(started_cluster)
catalog.create_namespace(namespace)
table = create_table(catalog, namespace, table_name)
table.append(pa.Table.from_pylist([generate_record()]))

create_clickhouse_iceberg_database(started_cluster, node1, CATALOG_NAME)
create_clickhouse_iceberg_database(started_cluster, node2, CATALOG_NAME)

query_id = f"object_storage_cluster_secondary_{uuid.uuid4().hex}"
assert (
node1.query(
f"SELECT symbol FROM remote('node2', '{CATALOG_NAME}', '{namespace}.{table_name}') "
"SETTINGS object_storage_cluster = 'cluster_simple'",
query_id=query_id,
)
== "kek\n"
)

node2.query("SYSTEM FLUSH LOGS system.query_log")
assert (
node2.query(
"SELECT ProfileEvents['DataLakeRestCatalogGetTableMetadata'] "
"FROM system.query_log "
f"WHERE initial_query_id = '{query_id}' AND type = 'QueryFinish' "
"AND startsWith(query, 'SELECT') "
"AND position(query, 'iceberg') = 0 "
"AND position(query, 'system.query_log') = 0"
)
== "1\n"
)

node2.query(f"DROP DATABASE IF EXISTS {CATALOG_NAME}")


def test_on_cluster_ddl_rejected_for_datalake_catalog(started_cluster):
node1 = started_cluster.instances["node1"]
node2 = started_cluster.instances["node2"]
Expand Down
Loading