Repository navigation
Read Iceberg catalog once during cluster requests. #2729
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: antalya-26.8
Are you sure you want to change the base?
Changes from all commits
e846234
dbe0f46
f281aab
6a9a4cb
2a4e242
b2f917c
fc2959e
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -203,6 +203,10 @@ class StorageObjectStorageCluster : public IStorageCluster | |
|
|
||
| /// Whether this query is served by `pure_storage` instead of being distributed over a cluster. | ||
| bool readsFromPureStorage(ContextPtr context) const; | ||
| /// Only `DataLakeCatalog` tables may reuse the resolved query tree for the header. Re-analyzing their | ||
| /// AST resolves the table through the REST catalog again. Other cluster reads keep re-analyzing the AST, | ||
| /// so their headers do not change. | ||
| bool canReuseResolvedQueryTreeForHeader() const override { return is_datalake_catalog_table; } | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why only catalogs can do it? |
||
|
|
||
| void readFallBackToPure( | ||
| QueryPlan & query_plan, | ||
|
|
@@ -243,6 +247,8 @@ class StorageObjectStorageCluster : public IStorageCluster | |
| const ObjectStoragePtr object_storage; | ||
| /// True for explicit *Cluster(...); false for SETTINGS / ENGINE / DataLake paths. | ||
| bool cluster_name_from_function_argument = false; | ||
| /// True only for tables resolved through a `DataLakeCatalog` database. | ||
| bool is_datalake_catalog_table = false; | ||
|
|
||
| /// non-clustered storage to fall back on pure realisation if needed | ||
| std::shared_ptr<StorageObjectStorage> pure_storage; | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -1441,6 +1441,201 @@ def test_cluster_select(started_cluster): | |
| assert node2.query(f"SELECT * FROM {CATALOG_NAME}.`{root_namespace}.{table_name}`", settings={"parallel_replicas_for_cluster_engines": 1, "enable_parallel_replicas": 2, "cluster_for_parallel_replicas": "cluster_simple"}) == 'pablo\n' | ||
|
|
||
|
|
||
| def test_object_storage_cluster_loads_rest_catalog_table_once(started_cluster): | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🤖 [Posted by an LLM agent on behalf of @mkmkme] These three tests only check |
||
| node = started_cluster.instances["node1"] | ||
|
|
||
| test_ref = f"test_object_storage_cluster_{uuid.uuid4().hex}" | ||
| table_name = f"{test_ref}_table" | ||
| namespace = f"{test_ref}_namespace" | ||
|
|
||
| catalog = load_catalog_impl(started_cluster) | ||
| catalog.create_namespace(namespace) | ||
| table = create_table(catalog, namespace, table_name) | ||
| table.append(pa.Table.from_pylist([generate_record()])) | ||
|
|
||
| create_clickhouse_iceberg_database(started_cluster, node, CATALOG_NAME) | ||
|
|
||
| query_id = f"object_storage_cluster_catalog_{uuid.uuid4().hex}" | ||
| assert ( | ||
| node.query( | ||
| f"SELECT * FROM {CATALOG_NAME}.`{namespace}.{table_name}` " | ||
| "SETTINGS object_storage_cluster = 'cluster_simple' FORMAT Null", | ||
| query_id=query_id, | ||
| ) | ||
| == "" | ||
| ) | ||
|
|
||
| node.query("SYSTEM FLUSH LOGS system.query_log") | ||
| assert ( | ||
| node.query( | ||
| "SELECT ProfileEvents['DataLakeRestCatalogGetTableMetadata'] " | ||
| "FROM system.query_log " | ||
| f"WHERE query_id = '{query_id}' AND type = 'QueryFinish' AND is_initial_query" | ||
| ) | ||
| == "1\n" | ||
| ) | ||
|
|
||
|
|
||
| def test_object_storage_cluster_join_loads_rest_catalog_table_once(started_cluster): | ||
| node = started_cluster.instances["node1"] | ||
|
|
||
| test_ref = f"test_object_storage_cluster_join_{uuid.uuid4().hex}" | ||
| table_name = f"{test_ref}_table" | ||
| namespace = f"{test_ref}_namespace" | ||
| memory_table = f"{test_ref}_memory" | ||
|
|
||
| catalog = load_catalog_impl(started_cluster) | ||
| catalog.create_namespace(namespace) | ||
| table = create_table(catalog, namespace, table_name) | ||
| table.append(pa.Table.from_pylist([generate_record()])) | ||
|
|
||
| create_clickhouse_iceberg_database(started_cluster, node, CATALOG_NAME) | ||
| node.query(f"CREATE TABLE {memory_table} (symbol String) ENGINE = Memory") | ||
| node.query(f"INSERT INTO {memory_table} VALUES ('kek')") | ||
|
|
||
| query_id = f"object_storage_cluster_join_catalog_{uuid.uuid4().hex}" | ||
| assert ( | ||
| node.query( | ||
| f"SELECT t.symbol FROM {CATALOG_NAME}.`{namespace}.{table_name}` AS t " | ||
| f"INNER JOIN {memory_table} AS m ON t.symbol = m.symbol " | ||
| "SETTINGS object_storage_cluster = 'cluster_simple'", | ||
| query_id=query_id, | ||
| ) | ||
| == "kek\n" | ||
| ) | ||
|
|
||
| node.query("SYSTEM FLUSH LOGS system.query_log") | ||
| assert ( | ||
| node.query( | ||
| "SELECT ProfileEvents['DataLakeRestCatalogGetTableMetadata'] " | ||
| "FROM system.query_log " | ||
| f"WHERE query_id = '{query_id}' AND type = 'QueryFinish' AND is_initial_query" | ||
| ) | ||
| == "1\n" | ||
| ) | ||
|
|
||
|
|
||
| def test_object_storage_cluster_right_join_loads_rest_catalog_table_once(started_cluster): | ||
| node = started_cluster.instances["node1"] | ||
|
|
||
| test_ref = f"test_object_storage_cluster_right_join_{uuid.uuid4().hex}" | ||
| table_name = f"{test_ref}_table" | ||
| namespace = f"{test_ref}_namespace" | ||
| memory_table = f"{test_ref}_memory" | ||
|
|
||
| catalog = load_catalog_impl(started_cluster) | ||
| catalog.create_namespace(namespace) | ||
| table = create_table(catalog, namespace, table_name) | ||
| table.append(pa.Table.from_pylist([generate_record()])) | ||
|
|
||
| create_clickhouse_iceberg_database(started_cluster, node, CATALOG_NAME) | ||
| node.query(f"CREATE TABLE {memory_table} (symbol String) ENGINE = Memory") | ||
| node.query(f"INSERT INTO {memory_table} VALUES ('kek')") | ||
|
|
||
| query_id = f"object_storage_cluster_right_join_catalog_{uuid.uuid4().hex}" | ||
| assert ( | ||
| node.query( | ||
| f"SELECT t.symbol FROM {memory_table} AS m " | ||
| f"INNER JOIN {CATALOG_NAME}.`{namespace}.{table_name}` AS t ON m.symbol = t.symbol " | ||
| "SETTINGS object_storage_cluster = 'cluster_simple'", | ||
| query_id=query_id, | ||
| ) | ||
| == "kek\n" | ||
| ) | ||
|
|
||
| node.query("SYSTEM FLUSH LOGS system.query_log") | ||
| assert ( | ||
| node.query( | ||
| "SELECT ProfileEvents['DataLakeRestCatalogGetTableMetadata'] " | ||
| "FROM system.query_log " | ||
| f"WHERE query_id = '{query_id}' AND type = 'QueryFinish' AND is_initial_query" | ||
| ) | ||
| == "1\n" | ||
| ) | ||
|
|
||
|
|
||
| def test_object_storage_cluster_right_in_subquery_loads_rest_catalog_table_once(started_cluster): | ||
| node = started_cluster.instances["node1"] | ||
|
|
||
| test_ref = f"test_object_storage_cluster_right_in_{uuid.uuid4().hex}" | ||
| table_name = f"{test_ref}_table" | ||
| namespace = f"{test_ref}_namespace" | ||
| memory_table = f"{test_ref}_memory" | ||
|
|
||
| catalog = load_catalog_impl(started_cluster) | ||
| catalog.create_namespace(namespace) | ||
| table = create_table(catalog, namespace, table_name) | ||
| table.append(pa.Table.from_pylist([generate_record()])) | ||
|
|
||
| create_clickhouse_iceberg_database(started_cluster, node, CATALOG_NAME) | ||
| node.query(f"CREATE TABLE {memory_table} (symbol String) ENGINE = Memory") | ||
| node.query(f"INSERT INTO {memory_table} VALUES ('kek')") | ||
|
|
||
| query_id = f"object_storage_cluster_right_in_catalog_{uuid.uuid4().hex}" | ||
| assert ( | ||
| node.query( | ||
| f"SELECT t.symbol FROM {memory_table} AS m " | ||
| f"INNER JOIN {CATALOG_NAME}.`{namespace}.{table_name}` AS t ON m.symbol = t.symbol " | ||
| "WHERE t.symbol IN (SELECT 'kek' FROM system.one) " | ||
| "SETTINGS object_storage_cluster = 'cluster_simple'", | ||
| query_id=query_id, | ||
| ) | ||
| == "kek\n" | ||
| ) | ||
|
|
||
| node.query("SYSTEM FLUSH LOGS system.query_log") | ||
| assert ( | ||
| node.query( | ||
| "SELECT ProfileEvents['DataLakeRestCatalogGetTableMetadata'] " | ||
| "FROM system.query_log " | ||
| f"WHERE query_id = '{query_id}' AND type = 'QueryFinish' AND is_initial_query" | ||
| ) | ||
| == "1\n" | ||
| ) | ||
|
|
||
|
|
||
| def test_object_storage_cluster_secondary_query_returns_shard_identifiers(started_cluster): | ||
| node1 = started_cluster.instances["node1"] | ||
| node2 = started_cluster.instances["node2"] | ||
|
|
||
| test_ref = f"test_object_storage_cluster_secondary_{uuid.uuid4().hex}" | ||
| table_name = f"{test_ref}_table" | ||
| namespace = f"{test_ref}_namespace" | ||
|
|
||
| catalog = load_catalog_impl(started_cluster) | ||
| catalog.create_namespace(namespace) | ||
| table = create_table(catalog, namespace, table_name) | ||
| table.append(pa.Table.from_pylist([generate_record()])) | ||
|
|
||
| create_clickhouse_iceberg_database(started_cluster, node1, CATALOG_NAME) | ||
| create_clickhouse_iceberg_database(started_cluster, node2, CATALOG_NAME) | ||
|
|
||
| query_id = f"object_storage_cluster_secondary_{uuid.uuid4().hex}" | ||
| assert ( | ||
| node1.query( | ||
| f"SELECT symbol FROM remote('node2', '{CATALOG_NAME}', '{namespace}.{table_name}') " | ||
| "SETTINGS object_storage_cluster = 'cluster_simple'", | ||
| query_id=query_id, | ||
| ) | ||
| == "kek\n" | ||
| ) | ||
|
|
||
| node2.query("SYSTEM FLUSH LOGS system.query_log") | ||
| assert ( | ||
| node2.query( | ||
| "SELECT ProfileEvents['DataLakeRestCatalogGetTableMetadata'] " | ||
| "FROM system.query_log " | ||
| f"WHERE initial_query_id = '{query_id}' AND type = 'QueryFinish' " | ||
| "AND startsWith(query, 'SELECT') " | ||
| "AND position(query, 'iceberg') = 0 " | ||
| "AND position(query, 'system.query_log') = 0" | ||
| ) | ||
| == "1\n" | ||
| ) | ||
|
|
||
| node2.query(f"DROP DATABASE IF EXISTS {CATALOG_NAME}") | ||
|
|
||
|
|
||
| def test_on_cluster_ddl_rejected_for_datalake_catalog(started_cluster): | ||
| node1 = started_cluster.instances["node1"] | ||
| node2 = started_cluster.instances["node2"] | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Nit: I would make this comment a bit more descriptive, something along the following lines:
I still don`t understand case 2 very well tho, looking into it