From 7667c85a2b82cf5dffbfd321387bf4a5e799ad38 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 6 Nov 2026 21:49:20 +0000 Subject: [PATCH] Merge pull request #128217 from groeneai/fix-partition-by-stateful-function-spans-chunks Evaluate the `PARTITION BY` expression of file and object storage writes once per INSERT, not once per block --- src/Storages/IPartitionStrategy.cpp | 30 +++++++++---------- src/Storages/IPartitionStrategy.h | 10 +++++-- src/Storages/PartitionedSink.cpp | 3 +- src/Storages/PartitionedSink.h | 1 + ...y_stateful_function_spans_chunks.reference | 11 +++++++ ...tion_by_stateful_function_spans_chunks.sql | 25 ++++++++++++++++ 6 files changed, 60 insertions(+), 20 deletions(-) create mode 100644 tests/queries/0_stateless/05459_partition_by_stateful_function_spans_chunks.reference create mode 100644 tests/queries/0_stateless/05459_partition_by_stateful_function_spans_chunks.sql diff --git a/src/Storages/IPartitionStrategy.cpp b/src/Storages/IPartitionStrategy.cpp index 079e5f07ecead..8906ded9056b4 100644 --- a/src/Storages/IPartitionStrategy.cpp +++ b/src/Storages/IPartitionStrategy.cpp @@ -241,6 +241,16 @@ IPartitionStrategy::getPartitionExpressionActions(ASTPtr & expression_ast) const return result; } +ColumnPtr IPartitionStrategy::computePartitionKey( + const Chunk & chunk, const PartitionExpressionActionsAndColumnName & partition_key_actions) const +{ + Block block_with_partition_by_expr = sample_block.cloneWithoutColumns(); + block_with_partition_by_expr.setColumns(chunk.getColumns()); + partition_key_actions.actions->execute(block_with_partition_by_expr); + + return block_with_partition_by_expr.getByName(partition_key_actions.column_name).column; +} + std::shared_ptr PartitionStrategyFactory::get(StrategyType strategy, ASTPtr partition_by, const NamesAndTypesList & partition_columns, @@ -298,18 +308,12 @@ WildcardPartitionStrategy::WildcardPartitionStrategy(KeyDescription partition_ke cacheDeterministicActions(cached_result, actions_with_column); } -ColumnPtr WildcardPartitionStrategy::computePartitionKey(const Chunk & chunk) const +IPartitionStrategy::PartitionExpressionActionsAndColumnName WildcardPartitionStrategy::getPartitionKeyActions() const { - auto actions_with_column = getCachedOrBuildActions( + return getCachedOrBuildActions( cached_result, *this, [&] { return buildToStringPartitionAST(partition_key_description.definition_ast); }); - - Block block_with_partition_by_expr = sample_block.cloneWithoutColumns(); - block_with_partition_by_expr.setColumns(chunk.getColumns()); - actions_with_column.actions->execute(block_with_partition_by_expr); - - return block_with_partition_by_expr.getByName(actions_with_column.column_name).column; } std::string WildcardPartitionStrategy::getPathForRead( @@ -385,18 +389,12 @@ std::string HiveStylePartitionStrategy::getPathForWrite( return path; } -ColumnPtr HiveStylePartitionStrategy::computePartitionKey(const Chunk & chunk) const +IPartitionStrategy::PartitionExpressionActionsAndColumnName HiveStylePartitionStrategy::getPartitionKeyActions() const { - auto actions_with_column = getCachedOrBuildActions( + return getCachedOrBuildActions( cached_result, *this, [&] { return buildHivePartitionAST(partition_key_description.definition_ast, getPartitionColumns()); }); - - Block block_with_partition_by_expr = sample_block.cloneWithoutColumns(); - block_with_partition_by_expr.setColumns(chunk.getColumns()); - actions_with_column.actions->execute(block_with_partition_by_expr); - - return block_with_partition_by_expr.getByName(actions_with_column.column_name).column; } ColumnRawPtrs HiveStylePartitionStrategy::getFormatChunkColumns(const Chunk & chunk) diff --git a/src/Storages/IPartitionStrategy.h b/src/Storages/IPartitionStrategy.h index b2899b0e4d0a7..b9e3e7301bc27 100644 --- a/src/Storages/IPartitionStrategy.h +++ b/src/Storages/IPartitionStrategy.h @@ -27,7 +27,11 @@ struct IPartitionStrategy virtual ~IPartitionStrategy() = default; - virtual ColumnPtr computePartitionKey(const Chunk & chunk) const = 0; + /// Rebuilt on every call if non-deterministic. A sink calls it once and reuses the result for all its chunks, + /// so a stateful function keeps its state across the chunks of one INSERT. + virtual PartitionExpressionActionsAndColumnName getPartitionKeyActions() const = 0; + + ColumnPtr computePartitionKey(const Chunk & chunk, const PartitionExpressionActionsAndColumnName & partition_key_actions) const; virtual std::string getPathForRead(const std::string & prefix) = 0; virtual std::string getPathForWrite(const std::string & prefix, const std::string & partition_key) = 0; @@ -92,7 +96,7 @@ struct WildcardPartitionStrategy : IPartitionStrategy { WildcardPartitionStrategy(KeyDescription partition_key_description_, const Block & sample_block_, ContextPtr context_); - ColumnPtr computePartitionKey(const Chunk & chunk) const override; + PartitionExpressionActionsAndColumnName getPartitionKeyActions() const override; std::string getPathForRead(const std::string & prefix) override; std::string getPathForWrite(const std::string & prefix, const std::string & partition_key) override; }; @@ -111,7 +115,7 @@ struct HiveStylePartitionStrategy : IPartitionStrategy const std::string & file_format_, bool partition_columns_in_data_file_); - ColumnPtr computePartitionKey(const Chunk & chunk) const override; + PartitionExpressionActionsAndColumnName getPartitionKeyActions() const override; std::string getPathForRead(const std::string & prefix) override; std::string getPathForWrite(const std::string & prefix, const std::string & partition_key) override; diff --git a/src/Storages/PartitionedSink.cpp b/src/Storages/PartitionedSink.cpp index 8af531716d4e9..4a0be70415e52 100644 --- a/src/Storages/PartitionedSink.cpp +++ b/src/Storages/PartitionedSink.cpp @@ -32,6 +32,7 @@ PartitionedSink::PartitionedSink( , partition_strategy(partition_strategy_) , context(context_) , source_header(source_header_) + , partition_key_actions(partition_strategy->getPartitionKeyActions()) { } @@ -50,7 +51,7 @@ SinkPtr PartitionedSink::getSinkForPartitionKey(std::string_view partition_key) void PartitionedSink::consume(Chunk & source_chunk) { - const ColumnPtr partition_by_result_column = partition_strategy->computePartitionKey(source_chunk); + const ColumnPtr partition_by_result_column = partition_strategy->computePartitionKey(source_chunk, partition_key_actions); /// Not all columns are serialized using the format writer (e.g, hive partitioning stores partition columns in the file path) const auto columns_to_consume = partition_strategy->getFormatChunkColumns(source_chunk); diff --git a/src/Storages/PartitionedSink.h b/src/Storages/PartitionedSink.h index af2b88baad277..90e02ba42a0dc 100644 --- a/src/Storages/PartitionedSink.h +++ b/src/Storages/PartitionedSink.h @@ -45,6 +45,7 @@ class PartitionedSink : public SinkToStorage private: ContextPtr context; SharedHeader source_header; + IPartitionStrategy::PartitionExpressionActionsAndColumnName partition_key_actions; absl::flat_hash_map partition_id_to_sink; HashMapWithSavedHash partition_id_to_chunk_index; diff --git a/tests/queries/0_stateless/05459_partition_by_stateful_function_spans_chunks.reference b/tests/queries/0_stateless/05459_partition_by_stateful_function_spans_chunks.reference new file mode 100644 index 0000000000000..61a6ea609e207 --- /dev/null +++ b/tests/queries/0_stateless/05459_partition_by_stateful_function_spans_chunks.reference @@ -0,0 +1,11 @@ +05459_rn_0.csv 1000 0 999 +05459_rn_1.csv 1000 1000 1999 +05459_rn_2.csv 1000 2000 2999 +05459_rn_3.csv 1000 3000 3999 +05459_rn_4.csv 1000 4000 4999 +05459_eng_0.csv 1000 10000 10999 +05459_eng_1.csv 1000 11000 11999 +05459_eng_2.csv 1000 2000 2999 +05459_eng_3.csv 1000 3000 3999 +05459_eng_4.csv 1000 4000 4999 +1 5000 diff --git a/tests/queries/0_stateless/05459_partition_by_stateful_function_spans_chunks.sql b/tests/queries/0_stateless/05459_partition_by_stateful_function_spans_chunks.sql new file mode 100644 index 0000000000000..54420711c58df --- /dev/null +++ b/tests/queries/0_stateless/05459_partition_by_stateful_function_spans_chunks.sql @@ -0,0 +1,25 @@ +-- Tags: no-fasttest +-- Tag no-fasttest: uses S3 + +SET max_block_size = 1000, max_insert_threads = 1, engine_file_truncate_on_insert = 1; + +-- file(): the counter spans the five 1000-row chunks +INSERT INTO FUNCTION file(currentDatabase() || '/05459_rn_{_partition_id}.csv', 'CSV', 'n UInt64') +PARTITION BY intDiv(rowNumberInAllBlocks(), 1000) +SELECT number FROM numbers(5000); +SELECT _file, count(), min(n), max(n) FROM file(currentDatabase() || '/05459_rn_*.csv', 'CSV', 'n UInt64') GROUP BY _file ORDER BY _file; + +-- S3 engine (strategy shared by all INSERTs into the table): spans chunks, starts anew with the next INSERT +CREATE TABLE t_05459 (n UInt64) +ENGINE = S3(s3_conn, filename = currentDatabase() || '/05459_eng_{_partition_id}.csv', format = CSV, partition_strategy = 'wildcard') +PARTITION BY intDiv(rowNumberInAllBlocks(), 1000); +INSERT INTO t_05459 SELECT number FROM numbers(5000) SETTINGS s3_truncate_on_insert = 1, s3_create_new_file_on_insert = 0; +INSERT INTO t_05459 SELECT number + 10000 FROM numbers(2000) SETTINGS s3_truncate_on_insert = 1, s3_create_new_file_on_insert = 0; +SELECT _file, count(), min(n), max(n) FROM s3(s3_conn, filename = currentDatabase() || '/05459_eng_*.csv', format = CSV, structure = 'n UInt64') GROUP BY _file ORDER BY _file; +DROP TABLE t_05459; + +-- now64() is evaluated once per INSERT, not once per chunk +INSERT INTO FUNCTION file(currentDatabase() || '/05459_now_{_partition_id}.csv', 'CSV', 'n UInt64') +PARTITION BY toUnixTimestamp64Nano(now64(9)) +SELECT number FROM numbers(5000); +SELECT count(DISTINCT _file), count() FROM file(currentDatabase() || '/05459_now_*.csv', 'CSV', 'n UInt64');