From dcd02697699b9cb3adcf2110b91b76915e466db2 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 6 Nov 2026 21:49:20 +0000 Subject: [PATCH 1/2] Merge pull request #128217 from groeneai/fix-partition-by-stateful-function-spans-chunks Evaluate the `PARTITION BY` expression of file and object storage writes once per INSERT, not once per block Conflicts committed as-is; the next commit resolves them: - src/Storages/IPartitionStrategy.cpp (both modified) - src/Storages/IPartitionStrategy.h (both modified) --- src/Storages/IPartitionStrategy.cpp | 28 +++++++++++++++++++ src/Storages/IPartitionStrategy.h | 16 +++++++++++ src/Storages/PartitionedSink.cpp | 3 +- src/Storages/PartitionedSink.h | 1 + ...y_stateful_function_spans_chunks.reference | 11 ++++++++ ...tion_by_stateful_function_spans_chunks.sql | 25 +++++++++++++++++ 6 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/05459_partition_by_stateful_function_spans_chunks.reference create mode 100644 tests/queries/0_stateless/05459_partition_by_stateful_function_spans_chunks.sql diff --git a/src/Storages/IPartitionStrategy.cpp b/src/Storages/IPartitionStrategy.cpp index 9e3c12d2b6ae2..06642667de661 100644 --- a/src/Storages/IPartitionStrategy.cpp +++ b/src/Storages/IPartitionStrategy.cpp @@ -216,6 +216,16 @@ IPartitionStrategy::getPartitionExpressionActions(ASTPtr & expression_ast) return result; } +ColumnPtr IPartitionStrategy::computePartitionKey( + const Chunk & chunk, const PartitionExpressionActionsAndColumnName & partition_key_actions) const +{ + Block block_with_partition_by_expr = sample_block.cloneWithoutColumns(); + block_with_partition_by_expr.setColumns(chunk.getColumns()); + partition_key_actions.actions->execute(block_with_partition_by_expr); + + return block_with_partition_by_expr.getByName(partition_key_actions.column_name).column; +} + std::shared_ptr PartitionStrategyFactory::get(StrategyType strategy, ASTPtr partition_by, const NamesAndTypesList & partition_columns, @@ -268,6 +278,7 @@ WildcardPartitionStrategy::WildcardPartitionStrategy(KeyDescription partition_ke { } +<<<<<<< HEAD ColumnPtr WildcardPartitionStrategy::computePartitionKey(const Chunk & chunk) { ASTs arguments(1, partition_key_description.definition_ast); @@ -279,6 +290,14 @@ ColumnPtr WildcardPartitionStrategy::computePartitionKey(const Chunk & chunk) actions_with_column.actions->execute(block_with_partition_by_expr); return block_with_partition_by_expr.getByName(actions_with_column.column_name).column; +======= +IPartitionStrategy::PartitionExpressionActionsAndColumnName WildcardPartitionStrategy::getPartitionKeyActions() const +{ + return getCachedOrBuildActions( + cached_result, + *this, + [&] { return buildToStringPartitionAST(partition_key_description.definition_ast); }); +>>>>>>> 690d3be35e1 (Merge pull request #128217 from groeneai/fix-partition-by-stateful-function-spans-chunks) } std::string WildcardPartitionStrategy::getPathForRead( @@ -348,6 +367,7 @@ std::string HiveStylePartitionStrategy::getPathForWrite( return path; } +<<<<<<< HEAD ColumnPtr HiveStylePartitionStrategy::computePartitionKey(const Chunk & chunk) { auto hive_ast = buildHivePartitionAST(partition_key_description.definition_ast, getPartitionColumns()); @@ -358,6 +378,14 @@ ColumnPtr HiveStylePartitionStrategy::computePartitionKey(const Chunk & chunk) actions_with_column.actions->execute(block_with_partition_by_expr); return block_with_partition_by_expr.getByName(actions_with_column.column_name).column; +======= +IPartitionStrategy::PartitionExpressionActionsAndColumnName HiveStylePartitionStrategy::getPartitionKeyActions() const +{ + return getCachedOrBuildActions( + cached_result, + *this, + [&] { return buildHivePartitionAST(partition_key_description.definition_ast, getPartitionColumns()); }); +>>>>>>> 690d3be35e1 (Merge pull request #128217 from groeneai/fix-partition-by-stateful-function-spans-chunks) } ColumnRawPtrs HiveStylePartitionStrategy::getFormatChunkColumns(const Chunk & chunk) diff --git a/src/Storages/IPartitionStrategy.h b/src/Storages/IPartitionStrategy.h index a0f5f4cdf9d39..da180f8f64057 100644 --- a/src/Storages/IPartitionStrategy.h +++ b/src/Storages/IPartitionStrategy.h @@ -27,7 +27,15 @@ struct IPartitionStrategy virtual ~IPartitionStrategy() = default; +<<<<<<< HEAD virtual ColumnPtr computePartitionKey(const Chunk & chunk) = 0; +======= + /// Rebuilt on every call if non-deterministic. A sink calls it once and reuses the result for all its chunks, + /// so a stateful function keeps its state across the chunks of one INSERT. + virtual PartitionExpressionActionsAndColumnName getPartitionKeyActions() const = 0; + + ColumnPtr computePartitionKey(const Chunk & chunk, const PartitionExpressionActionsAndColumnName & partition_key_actions) const; +>>>>>>> 690d3be35e1 (Merge pull request #128217 from groeneai/fix-partition-by-stateful-function-spans-chunks) virtual std::string getPathForRead(const std::string & prefix) = 0; virtual std::string getPathForWrite(const std::string & prefix, const std::string & partition_key) = 0; @@ -92,7 +100,11 @@ struct WildcardPartitionStrategy : IPartitionStrategy { WildcardPartitionStrategy(KeyDescription partition_key_description_, const Block & sample_block_, ContextPtr context_); +<<<<<<< HEAD ColumnPtr computePartitionKey(const Chunk & chunk) override; +======= + PartitionExpressionActionsAndColumnName getPartitionKeyActions() const override; +>>>>>>> 690d3be35e1 (Merge pull request #128217 from groeneai/fix-partition-by-stateful-function-spans-chunks) std::string getPathForRead(const std::string & prefix) override; std::string getPathForWrite(const std::string & prefix, const std::string & partition_key) override; }; @@ -111,7 +123,11 @@ struct HiveStylePartitionStrategy : IPartitionStrategy const std::string & file_format_, bool partition_columns_in_data_file_); +<<<<<<< HEAD ColumnPtr computePartitionKey(const Chunk & chunk) override; +======= + PartitionExpressionActionsAndColumnName getPartitionKeyActions() const override; +>>>>>>> 690d3be35e1 (Merge pull request #128217 from groeneai/fix-partition-by-stateful-function-spans-chunks) std::string getPathForRead(const std::string & prefix) override; std::string getPathForWrite(const std::string & prefix, const std::string & partition_key) override; diff --git a/src/Storages/PartitionedSink.cpp b/src/Storages/PartitionedSink.cpp index 61789c3db19a8..cabda041978ca 100644 --- a/src/Storages/PartitionedSink.cpp +++ b/src/Storages/PartitionedSink.cpp @@ -32,6 +32,7 @@ PartitionedSink::PartitionedSink( , partition_strategy(partition_strategy_) , context(context_) , source_header(source_header_) + , partition_key_actions(partition_strategy->getPartitionKeyActions()) { } @@ -50,7 +51,7 @@ SinkPtr PartitionedSink::getSinkForPartitionKey(std::string_view partition_key) void PartitionedSink::consume(Chunk & source_chunk) { - const ColumnPtr partition_by_result_column = partition_strategy->computePartitionKey(source_chunk); + const ColumnPtr partition_by_result_column = partition_strategy->computePartitionKey(source_chunk, partition_key_actions); /// Not all columns are serialized using the format writer (e.g, hive partitioning stores partition columns in the file path) const auto columns_to_consume = partition_strategy->getFormatChunkColumns(source_chunk); diff --git a/src/Storages/PartitionedSink.h b/src/Storages/PartitionedSink.h index 3aa9cf6db16ae..0a44de7f53d3a 100644 --- a/src/Storages/PartitionedSink.h +++ b/src/Storages/PartitionedSink.h @@ -46,6 +46,7 @@ class PartitionedSink : public SinkToStorage private: ContextPtr context; SharedHeader source_header; + IPartitionStrategy::PartitionExpressionActionsAndColumnName partition_key_actions; absl::flat_hash_map partition_id_to_sink; HashMapWithSavedHash partition_id_to_chunk_index; diff --git a/tests/queries/0_stateless/05459_partition_by_stateful_function_spans_chunks.reference b/tests/queries/0_stateless/05459_partition_by_stateful_function_spans_chunks.reference new file mode 100644 index 0000000000000..61a6ea609e207 --- /dev/null +++ b/tests/queries/0_stateless/05459_partition_by_stateful_function_spans_chunks.reference @@ -0,0 +1,11 @@ +05459_rn_0.csv 1000 0 999 +05459_rn_1.csv 1000 1000 1999 +05459_rn_2.csv 1000 2000 2999 +05459_rn_3.csv 1000 3000 3999 +05459_rn_4.csv 1000 4000 4999 +05459_eng_0.csv 1000 10000 10999 +05459_eng_1.csv 1000 11000 11999 +05459_eng_2.csv 1000 2000 2999 +05459_eng_3.csv 1000 3000 3999 +05459_eng_4.csv 1000 4000 4999 +1 5000 diff --git a/tests/queries/0_stateless/05459_partition_by_stateful_function_spans_chunks.sql b/tests/queries/0_stateless/05459_partition_by_stateful_function_spans_chunks.sql new file mode 100644 index 0000000000000..54420711c58df --- /dev/null +++ b/tests/queries/0_stateless/05459_partition_by_stateful_function_spans_chunks.sql @@ -0,0 +1,25 @@ +-- Tags: no-fasttest +-- Tag no-fasttest: uses S3 + +SET max_block_size = 1000, max_insert_threads = 1, engine_file_truncate_on_insert = 1; + +-- file(): the counter spans the five 1000-row chunks +INSERT INTO FUNCTION file(currentDatabase() || '/05459_rn_{_partition_id}.csv', 'CSV', 'n UInt64') +PARTITION BY intDiv(rowNumberInAllBlocks(), 1000) +SELECT number FROM numbers(5000); +SELECT _file, count(), min(n), max(n) FROM file(currentDatabase() || '/05459_rn_*.csv', 'CSV', 'n UInt64') GROUP BY _file ORDER BY _file; + +-- S3 engine (strategy shared by all INSERTs into the table): spans chunks, starts anew with the next INSERT +CREATE TABLE t_05459 (n UInt64) +ENGINE = S3(s3_conn, filename = currentDatabase() || '/05459_eng_{_partition_id}.csv', format = CSV, partition_strategy = 'wildcard') +PARTITION BY intDiv(rowNumberInAllBlocks(), 1000); +INSERT INTO t_05459 SELECT number FROM numbers(5000) SETTINGS s3_truncate_on_insert = 1, s3_create_new_file_on_insert = 0; +INSERT INTO t_05459 SELECT number + 10000 FROM numbers(2000) SETTINGS s3_truncate_on_insert = 1, s3_create_new_file_on_insert = 0; +SELECT _file, count(), min(n), max(n) FROM s3(s3_conn, filename = currentDatabase() || '/05459_eng_*.csv', format = CSV, structure = 'n UInt64') GROUP BY _file ORDER BY _file; +DROP TABLE t_05459; + +-- now64() is evaluated once per INSERT, not once per chunk +INSERT INTO FUNCTION file(currentDatabase() || '/05459_now_{_partition_id}.csv', 'CSV', 'n UInt64') +PARTITION BY toUnixTimestamp64Nano(now64(9)) +SELECT number FROM numbers(5000); +SELECT count(DISTINCT _file), count() FROM file(currentDatabase() || '/05459_now_*.csv', 'CSV', 'n UInt64'); From 82ec992a8c10a846b021454084aa395210cb669f Mon Sep 17 00:00:00 2001 From: blau-ai <302218339+blau-ai@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:51:33 +0000 Subject: [PATCH 2/2] Resolve conflicts in backport of #128217 Adapted the PR to stable-26.3's non-const strategy API, which has no `getCachedOrBuildActions`, and kept a one-argument `computePartitionKey` for the DeltaLake sink. --- src/Storages/IPartitionStrategy.cpp | 43 ++++++----------------------- src/Storages/IPartitionStrategy.h | 21 ++++---------- 2 files changed, 15 insertions(+), 49 deletions(-) diff --git a/src/Storages/IPartitionStrategy.cpp b/src/Storages/IPartitionStrategy.cpp index 06642667de661..cc187661a5046 100644 --- a/src/Storages/IPartitionStrategy.cpp +++ b/src/Storages/IPartitionStrategy.cpp @@ -226,6 +226,11 @@ ColumnPtr IPartitionStrategy::computePartitionKey( return block_with_partition_by_expr.getByName(partition_key_actions.column_name).column; } +ColumnPtr IPartitionStrategy::computePartitionKey(const Chunk & chunk) +{ + return computePartitionKey(chunk, getPartitionKeyActions()); +} + std::shared_ptr PartitionStrategyFactory::get(StrategyType strategy, ASTPtr partition_by, const NamesAndTypesList & partition_columns, @@ -278,26 +283,11 @@ WildcardPartitionStrategy::WildcardPartitionStrategy(KeyDescription partition_ke { } -<<<<<<< HEAD -ColumnPtr WildcardPartitionStrategy::computePartitionKey(const Chunk & chunk) +IPartitionStrategy::PartitionExpressionActionsAndColumnName WildcardPartitionStrategy::getPartitionKeyActions() { ASTs arguments(1, partition_key_description.definition_ast); ASTPtr partition_by_string = makeASTFunction("toString", std::move(arguments)); - auto actions_with_column = getPartitionExpressionActions(partition_by_string); - - Block block_with_partition_by_expr = sample_block.cloneWithoutColumns(); - block_with_partition_by_expr.setColumns(chunk.getColumns()); - actions_with_column.actions->execute(block_with_partition_by_expr); - - return block_with_partition_by_expr.getByName(actions_with_column.column_name).column; -======= -IPartitionStrategy::PartitionExpressionActionsAndColumnName WildcardPartitionStrategy::getPartitionKeyActions() const -{ - return getCachedOrBuildActions( - cached_result, - *this, - [&] { return buildToStringPartitionAST(partition_key_description.definition_ast); }); ->>>>>>> 690d3be35e1 (Merge pull request #128217 from groeneai/fix-partition-by-stateful-function-spans-chunks) + return getPartitionExpressionActions(partition_by_string); } std::string WildcardPartitionStrategy::getPathForRead( @@ -367,25 +357,10 @@ std::string HiveStylePartitionStrategy::getPathForWrite( return path; } -<<<<<<< HEAD -ColumnPtr HiveStylePartitionStrategy::computePartitionKey(const Chunk & chunk) +IPartitionStrategy::PartitionExpressionActionsAndColumnName HiveStylePartitionStrategy::getPartitionKeyActions() { auto hive_ast = buildHivePartitionAST(partition_key_description.definition_ast, getPartitionColumns()); - auto actions_with_column = getPartitionExpressionActions(hive_ast); - - Block block_with_partition_by_expr = sample_block.cloneWithoutColumns(); - block_with_partition_by_expr.setColumns(chunk.getColumns()); - actions_with_column.actions->execute(block_with_partition_by_expr); - - return block_with_partition_by_expr.getByName(actions_with_column.column_name).column; -======= -IPartitionStrategy::PartitionExpressionActionsAndColumnName HiveStylePartitionStrategy::getPartitionKeyActions() const -{ - return getCachedOrBuildActions( - cached_result, - *this, - [&] { return buildHivePartitionAST(partition_key_description.definition_ast, getPartitionColumns()); }); ->>>>>>> 690d3be35e1 (Merge pull request #128217 from groeneai/fix-partition-by-stateful-function-spans-chunks) + return getPartitionExpressionActions(hive_ast); } ColumnRawPtrs HiveStylePartitionStrategy::getFormatChunkColumns(const Chunk & chunk) diff --git a/src/Storages/IPartitionStrategy.h b/src/Storages/IPartitionStrategy.h index da180f8f64057..cc203b3a51452 100644 --- a/src/Storages/IPartitionStrategy.h +++ b/src/Storages/IPartitionStrategy.h @@ -27,15 +27,14 @@ struct IPartitionStrategy virtual ~IPartitionStrategy() = default; -<<<<<<< HEAD - virtual ColumnPtr computePartitionKey(const Chunk & chunk) = 0; -======= /// Rebuilt on every call if non-deterministic. A sink calls it once and reuses the result for all its chunks, /// so a stateful function keeps its state across the chunks of one INSERT. - virtual PartitionExpressionActionsAndColumnName getPartitionKeyActions() const = 0; + virtual PartitionExpressionActionsAndColumnName getPartitionKeyActions() = 0; ColumnPtr computePartitionKey(const Chunk & chunk, const PartitionExpressionActionsAndColumnName & partition_key_actions) const; ->>>>>>> 690d3be35e1 (Merge pull request #128217 from groeneai/fix-partition-by-stateful-function-spans-chunks) + + /// Convenience overload: builds the actions for this call only. + ColumnPtr computePartitionKey(const Chunk & chunk); virtual std::string getPathForRead(const std::string & prefix) = 0; virtual std::string getPathForWrite(const std::string & prefix, const std::string & partition_key) = 0; @@ -100,11 +99,7 @@ struct WildcardPartitionStrategy : IPartitionStrategy { WildcardPartitionStrategy(KeyDescription partition_key_description_, const Block & sample_block_, ContextPtr context_); -<<<<<<< HEAD - ColumnPtr computePartitionKey(const Chunk & chunk) override; -======= - PartitionExpressionActionsAndColumnName getPartitionKeyActions() const override; ->>>>>>> 690d3be35e1 (Merge pull request #128217 from groeneai/fix-partition-by-stateful-function-spans-chunks) + PartitionExpressionActionsAndColumnName getPartitionKeyActions() override; std::string getPathForRead(const std::string & prefix) override; std::string getPathForWrite(const std::string & prefix, const std::string & partition_key) override; }; @@ -123,11 +118,7 @@ struct HiveStylePartitionStrategy : IPartitionStrategy const std::string & file_format_, bool partition_columns_in_data_file_); -<<<<<<< HEAD - ColumnPtr computePartitionKey(const Chunk & chunk) override; -======= - PartitionExpressionActionsAndColumnName getPartitionKeyActions() const override; ->>>>>>> 690d3be35e1 (Merge pull request #128217 from groeneai/fix-partition-by-stateful-function-spans-chunks) + PartitionExpressionActionsAndColumnName getPartitionKeyActions() override; std::string getPathForRead(const std::string & prefix) override; std::string getPathForWrite(const std::string & prefix, const std::string & partition_key) override;