diff --git a/src/Storages/IPartitionStrategy.cpp b/src/Storages/IPartitionStrategy.cpp index 9e3c12d2b6ae2..cc187661a5046 100644 --- a/src/Storages/IPartitionStrategy.cpp +++ b/src/Storages/IPartitionStrategy.cpp @@ -216,6 +216,21 @@ IPartitionStrategy::getPartitionExpressionActions(ASTPtr & expression_ast) return result; } +ColumnPtr IPartitionStrategy::computePartitionKey( + const Chunk & chunk, const PartitionExpressionActionsAndColumnName & partition_key_actions) const +{ + Block block_with_partition_by_expr = sample_block.cloneWithoutColumns(); + block_with_partition_by_expr.setColumns(chunk.getColumns()); + partition_key_actions.actions->execute(block_with_partition_by_expr); + + return block_with_partition_by_expr.getByName(partition_key_actions.column_name).column; +} + +ColumnPtr IPartitionStrategy::computePartitionKey(const Chunk & chunk) +{ + return computePartitionKey(chunk, getPartitionKeyActions()); +} + std::shared_ptr PartitionStrategyFactory::get(StrategyType strategy, ASTPtr partition_by, const NamesAndTypesList & partition_columns, @@ -268,17 +283,11 @@ WildcardPartitionStrategy::WildcardPartitionStrategy(KeyDescription partition_ke { } -ColumnPtr WildcardPartitionStrategy::computePartitionKey(const Chunk & chunk) +IPartitionStrategy::PartitionExpressionActionsAndColumnName WildcardPartitionStrategy::getPartitionKeyActions() { ASTs arguments(1, partition_key_description.definition_ast); ASTPtr partition_by_string = makeASTFunction("toString", std::move(arguments)); - auto actions_with_column = getPartitionExpressionActions(partition_by_string); - - Block block_with_partition_by_expr = sample_block.cloneWithoutColumns(); - block_with_partition_by_expr.setColumns(chunk.getColumns()); - actions_with_column.actions->execute(block_with_partition_by_expr); - - return block_with_partition_by_expr.getByName(actions_with_column.column_name).column; + return getPartitionExpressionActions(partition_by_string); } std::string WildcardPartitionStrategy::getPathForRead( @@ -348,16 +357,10 @@ std::string HiveStylePartitionStrategy::getPathForWrite( return path; } -ColumnPtr HiveStylePartitionStrategy::computePartitionKey(const Chunk & chunk) +IPartitionStrategy::PartitionExpressionActionsAndColumnName HiveStylePartitionStrategy::getPartitionKeyActions() { auto hive_ast = buildHivePartitionAST(partition_key_description.definition_ast, getPartitionColumns()); - auto actions_with_column = getPartitionExpressionActions(hive_ast); - - Block block_with_partition_by_expr = sample_block.cloneWithoutColumns(); - block_with_partition_by_expr.setColumns(chunk.getColumns()); - actions_with_column.actions->execute(block_with_partition_by_expr); - - return block_with_partition_by_expr.getByName(actions_with_column.column_name).column; + return getPartitionExpressionActions(hive_ast); } ColumnRawPtrs HiveStylePartitionStrategy::getFormatChunkColumns(const Chunk & chunk) diff --git a/src/Storages/IPartitionStrategy.h b/src/Storages/IPartitionStrategy.h index a0f5f4cdf9d39..cc203b3a51452 100644 --- a/src/Storages/IPartitionStrategy.h +++ b/src/Storages/IPartitionStrategy.h @@ -27,7 +27,14 @@ struct IPartitionStrategy virtual ~IPartitionStrategy() = default; - virtual ColumnPtr computePartitionKey(const Chunk & chunk) = 0; + /// Rebuilt on every call if non-deterministic. A sink calls it once and reuses the result for all its chunks, + /// so a stateful function keeps its state across the chunks of one INSERT. + virtual PartitionExpressionActionsAndColumnName getPartitionKeyActions() = 0; + + ColumnPtr computePartitionKey(const Chunk & chunk, const PartitionExpressionActionsAndColumnName & partition_key_actions) const; + + /// Convenience overload: builds the actions for this call only. + ColumnPtr computePartitionKey(const Chunk & chunk); virtual std::string getPathForRead(const std::string & prefix) = 0; virtual std::string getPathForWrite(const std::string & prefix, const std::string & partition_key) = 0; @@ -92,7 +99,7 @@ struct WildcardPartitionStrategy : IPartitionStrategy { WildcardPartitionStrategy(KeyDescription partition_key_description_, const Block & sample_block_, ContextPtr context_); - ColumnPtr computePartitionKey(const Chunk & chunk) override; + PartitionExpressionActionsAndColumnName getPartitionKeyActions() override; std::string getPathForRead(const std::string & prefix) override; std::string getPathForWrite(const std::string & prefix, const std::string & partition_key) override; }; @@ -111,7 +118,7 @@ struct HiveStylePartitionStrategy : IPartitionStrategy const std::string & file_format_, bool partition_columns_in_data_file_); - ColumnPtr computePartitionKey(const Chunk & chunk) override; + PartitionExpressionActionsAndColumnName getPartitionKeyActions() override; std::string getPathForRead(const std::string & prefix) override; std::string getPathForWrite(const std::string & prefix, const std::string & partition_key) override; diff --git a/src/Storages/PartitionedSink.cpp b/src/Storages/PartitionedSink.cpp index 61789c3db19a8..cabda041978ca 100644 --- a/src/Storages/PartitionedSink.cpp +++ b/src/Storages/PartitionedSink.cpp @@ -32,6 +32,7 @@ PartitionedSink::PartitionedSink( , partition_strategy(partition_strategy_) , context(context_) , source_header(source_header_) + , partition_key_actions(partition_strategy->getPartitionKeyActions()) { } @@ -50,7 +51,7 @@ SinkPtr PartitionedSink::getSinkForPartitionKey(std::string_view partition_key) void PartitionedSink::consume(Chunk & source_chunk) { - const ColumnPtr partition_by_result_column = partition_strategy->computePartitionKey(source_chunk); + const ColumnPtr partition_by_result_column = partition_strategy->computePartitionKey(source_chunk, partition_key_actions); /// Not all columns are serialized using the format writer (e.g, hive partitioning stores partition columns in the file path) const auto columns_to_consume = partition_strategy->getFormatChunkColumns(source_chunk); diff --git a/src/Storages/PartitionedSink.h b/src/Storages/PartitionedSink.h index 3aa9cf6db16ae..0a44de7f53d3a 100644 --- a/src/Storages/PartitionedSink.h +++ b/src/Storages/PartitionedSink.h @@ -46,6 +46,7 @@ class PartitionedSink : public SinkToStorage private: ContextPtr context; SharedHeader source_header; + IPartitionStrategy::PartitionExpressionActionsAndColumnName partition_key_actions; absl::flat_hash_map partition_id_to_sink; HashMapWithSavedHash partition_id_to_chunk_index; diff --git a/tests/queries/0_stateless/05459_partition_by_stateful_function_spans_chunks.reference b/tests/queries/0_stateless/05459_partition_by_stateful_function_spans_chunks.reference new file mode 100644 index 0000000000000..61a6ea609e207 --- /dev/null +++ b/tests/queries/0_stateless/05459_partition_by_stateful_function_spans_chunks.reference @@ -0,0 +1,11 @@ +05459_rn_0.csv 1000 0 999 +05459_rn_1.csv 1000 1000 1999 +05459_rn_2.csv 1000 2000 2999 +05459_rn_3.csv 1000 3000 3999 +05459_rn_4.csv 1000 4000 4999 +05459_eng_0.csv 1000 10000 10999 +05459_eng_1.csv 1000 11000 11999 +05459_eng_2.csv 1000 2000 2999 +05459_eng_3.csv 1000 3000 3999 +05459_eng_4.csv 1000 4000 4999 +1 5000 diff --git a/tests/queries/0_stateless/05459_partition_by_stateful_function_spans_chunks.sql b/tests/queries/0_stateless/05459_partition_by_stateful_function_spans_chunks.sql new file mode 100644 index 0000000000000..54420711c58df --- /dev/null +++ b/tests/queries/0_stateless/05459_partition_by_stateful_function_spans_chunks.sql @@ -0,0 +1,25 @@ +-- Tags: no-fasttest +-- Tag no-fasttest: uses S3 + +SET max_block_size = 1000, max_insert_threads = 1, engine_file_truncate_on_insert = 1; + +-- file(): the counter spans the five 1000-row chunks +INSERT INTO FUNCTION file(currentDatabase() || '/05459_rn_{_partition_id}.csv', 'CSV', 'n UInt64') +PARTITION BY intDiv(rowNumberInAllBlocks(), 1000) +SELECT number FROM numbers(5000); +SELECT _file, count(), min(n), max(n) FROM file(currentDatabase() || '/05459_rn_*.csv', 'CSV', 'n UInt64') GROUP BY _file ORDER BY _file; + +-- S3 engine (strategy shared by all INSERTs into the table): spans chunks, starts anew with the next INSERT +CREATE TABLE t_05459 (n UInt64) +ENGINE = S3(s3_conn, filename = currentDatabase() || '/05459_eng_{_partition_id}.csv', format = CSV, partition_strategy = 'wildcard') +PARTITION BY intDiv(rowNumberInAllBlocks(), 1000); +INSERT INTO t_05459 SELECT number FROM numbers(5000) SETTINGS s3_truncate_on_insert = 1, s3_create_new_file_on_insert = 0; +INSERT INTO t_05459 SELECT number + 10000 FROM numbers(2000) SETTINGS s3_truncate_on_insert = 1, s3_create_new_file_on_insert = 0; +SELECT _file, count(), min(n), max(n) FROM s3(s3_conn, filename = currentDatabase() || '/05459_eng_*.csv', format = CSV, structure = 'n UInt64') GROUP BY _file ORDER BY _file; +DROP TABLE t_05459; + +-- now64() is evaluated once per INSERT, not once per chunk +INSERT INTO FUNCTION file(currentDatabase() || '/05459_now_{_partition_id}.csv', 'CSV', 'n UInt64') +PARTITION BY toUnixTimestamp64Nano(now64(9)) +SELECT number FROM numbers(5000); +SELECT count(DISTINCT _file), count() FROM file(currentDatabase() || '/05459_now_*.csv', 'CSV', 'n UInt64');