Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 19 additions & 16 deletions src/Storages/IPartitionStrategy.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,21 @@ IPartitionStrategy::getPartitionExpressionActions(ASTPtr & expression_ast)
return result;
}

ColumnPtr IPartitionStrategy::computePartitionKey(
const Chunk & chunk, const PartitionExpressionActionsAndColumnName & partition_key_actions) const
{
Block block_with_partition_by_expr = sample_block.cloneWithoutColumns();
block_with_partition_by_expr.setColumns(chunk.getColumns());
partition_key_actions.actions->execute(block_with_partition_by_expr);

return block_with_partition_by_expr.getByName(partition_key_actions.column_name).column;
}

ColumnPtr IPartitionStrategy::computePartitionKey(const Chunk & chunk)
{
return computePartitionKey(chunk, getPartitionKeyActions());
}

std::shared_ptr<IPartitionStrategy> PartitionStrategyFactory::get(StrategyType strategy,
ASTPtr partition_by,
const NamesAndTypesList & partition_columns,
Expand Down Expand Up @@ -268,17 +283,11 @@ WildcardPartitionStrategy::WildcardPartitionStrategy(KeyDescription partition_ke
{
}

ColumnPtr WildcardPartitionStrategy::computePartitionKey(const Chunk & chunk)
IPartitionStrategy::PartitionExpressionActionsAndColumnName WildcardPartitionStrategy::getPartitionKeyActions()
{
ASTs arguments(1, partition_key_description.definition_ast);
ASTPtr partition_by_string = makeASTFunction("toString", std::move(arguments));
auto actions_with_column = getPartitionExpressionActions(partition_by_string);

Block block_with_partition_by_expr = sample_block.cloneWithoutColumns();
block_with_partition_by_expr.setColumns(chunk.getColumns());
actions_with_column.actions->execute(block_with_partition_by_expr);

return block_with_partition_by_expr.getByName(actions_with_column.column_name).column;
return getPartitionExpressionActions(partition_by_string);
}

std::string WildcardPartitionStrategy::getPathForRead(
Expand Down Expand Up @@ -348,16 +357,10 @@ std::string HiveStylePartitionStrategy::getPathForWrite(
return path;
}

ColumnPtr HiveStylePartitionStrategy::computePartitionKey(const Chunk & chunk)
IPartitionStrategy::PartitionExpressionActionsAndColumnName HiveStylePartitionStrategy::getPartitionKeyActions()
{
auto hive_ast = buildHivePartitionAST(partition_key_description.definition_ast, getPartitionColumns());
auto actions_with_column = getPartitionExpressionActions(hive_ast);

Block block_with_partition_by_expr = sample_block.cloneWithoutColumns();
block_with_partition_by_expr.setColumns(chunk.getColumns());
actions_with_column.actions->execute(block_with_partition_by_expr);

return block_with_partition_by_expr.getByName(actions_with_column.column_name).column;
return getPartitionExpressionActions(hive_ast);
}

ColumnRawPtrs HiveStylePartitionStrategy::getFormatChunkColumns(const Chunk & chunk)
Expand Down
13 changes: 10 additions & 3 deletions src/Storages/IPartitionStrategy.h
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,14 @@ struct IPartitionStrategy

virtual ~IPartitionStrategy() = default;

virtual ColumnPtr computePartitionKey(const Chunk & chunk) = 0;
/// Rebuilt on every call if non-deterministic. A sink calls it once and reuses the result for all its chunks,
/// so a stateful function keeps its state across the chunks of one INSERT.
virtual PartitionExpressionActionsAndColumnName getPartitionKeyActions() = 0;

ColumnPtr computePartitionKey(const Chunk & chunk, const PartitionExpressionActionsAndColumnName & partition_key_actions) const;

/// Convenience overload: builds the actions for this call only.
ColumnPtr computePartitionKey(const Chunk & chunk);

virtual std::string getPathForRead(const std::string & prefix) = 0;
virtual std::string getPathForWrite(const std::string & prefix, const std::string & partition_key) = 0;
Expand Down Expand Up @@ -92,7 +99,7 @@ struct WildcardPartitionStrategy : IPartitionStrategy
{
WildcardPartitionStrategy(KeyDescription partition_key_description_, const Block & sample_block_, ContextPtr context_);

ColumnPtr computePartitionKey(const Chunk & chunk) override;
PartitionExpressionActionsAndColumnName getPartitionKeyActions() override;
std::string getPathForRead(const std::string & prefix) override;
std::string getPathForWrite(const std::string & prefix, const std::string & partition_key) override;
};
Expand All @@ -111,7 +118,7 @@ struct HiveStylePartitionStrategy : IPartitionStrategy
const std::string & file_format_,
bool partition_columns_in_data_file_);

ColumnPtr computePartitionKey(const Chunk & chunk) override;
PartitionExpressionActionsAndColumnName getPartitionKeyActions() override;
std::string getPathForRead(const std::string & prefix) override;
std::string getPathForWrite(const std::string & prefix, const std::string & partition_key) override;

Expand Down
3 changes: 2 additions & 1 deletion src/Storages/PartitionedSink.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ PartitionedSink::PartitionedSink(
, partition_strategy(partition_strategy_)
, context(context_)
, source_header(source_header_)
, partition_key_actions(partition_strategy->getPartitionKeyActions())
{
}

Expand All @@ -50,7 +51,7 @@ SinkPtr PartitionedSink::getSinkForPartitionKey(std::string_view partition_key)

void PartitionedSink::consume(Chunk & source_chunk)
{
const ColumnPtr partition_by_result_column = partition_strategy->computePartitionKey(source_chunk);
const ColumnPtr partition_by_result_column = partition_strategy->computePartitionKey(source_chunk, partition_key_actions);

/// Not all columns are serialized using the format writer (e.g, hive partitioning stores partition columns in the file path)
const auto columns_to_consume = partition_strategy->getFormatChunkColumns(source_chunk);
Expand Down
1 change: 1 addition & 0 deletions src/Storages/PartitionedSink.h
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ class PartitionedSink : public SinkToStorage
private:
ContextPtr context;
SharedHeader source_header;
IPartitionStrategy::PartitionExpressionActionsAndColumnName partition_key_actions;

absl::flat_hash_map<std::string_view, SinkPtr> partition_id_to_sink;
HashMapWithSavedHash<std::string_view, size_t> partition_id_to_chunk_index;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
05459_rn_0.csv 1000 0 999
05459_rn_1.csv 1000 1000 1999
05459_rn_2.csv 1000 2000 2999
05459_rn_3.csv 1000 3000 3999
05459_rn_4.csv 1000 4000 4999
05459_eng_0.csv 1000 10000 10999
05459_eng_1.csv 1000 11000 11999
05459_eng_2.csv 1000 2000 2999
05459_eng_3.csv 1000 3000 3999
05459_eng_4.csv 1000 4000 4999
1 5000
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
-- Tags: no-fasttest
-- Tag no-fasttest: uses S3

SET max_block_size = 1000, max_insert_threads = 1, engine_file_truncate_on_insert = 1;

-- file(): the counter spans the five 1000-row chunks
INSERT INTO FUNCTION file(currentDatabase() || '/05459_rn_{_partition_id}.csv', 'CSV', 'n UInt64')
PARTITION BY intDiv(rowNumberInAllBlocks(), 1000)
SELECT number FROM numbers(5000);
SELECT _file, count(), min(n), max(n) FROM file(currentDatabase() || '/05459_rn_*.csv', 'CSV', 'n UInt64') GROUP BY _file ORDER BY _file;

-- S3 engine (strategy shared by all INSERTs into the table): spans chunks, starts anew with the next INSERT
CREATE TABLE t_05459 (n UInt64)
ENGINE = S3(s3_conn, filename = currentDatabase() || '/05459_eng_{_partition_id}.csv', format = CSV, partition_strategy = 'wildcard')
PARTITION BY intDiv(rowNumberInAllBlocks(), 1000);
INSERT INTO t_05459 SELECT number FROM numbers(5000) SETTINGS s3_truncate_on_insert = 1, s3_create_new_file_on_insert = 0;
INSERT INTO t_05459 SELECT number + 10000 FROM numbers(2000) SETTINGS s3_truncate_on_insert = 1, s3_create_new_file_on_insert = 0;
SELECT _file, count(), min(n), max(n) FROM s3(s3_conn, filename = currentDatabase() || '/05459_eng_*.csv', format = CSV, structure = 'n UInt64') GROUP BY _file ORDER BY _file;
DROP TABLE t_05459;

-- now64() is evaluated once per INSERT, not once per chunk
INSERT INTO FUNCTION file(currentDatabase() || '/05459_now_{_partition_id}.csv', 'CSV', 'n UInt64')
PARTITION BY toUnixTimestamp64Nano(now64(9))
SELECT number FROM numbers(5000);
SELECT count(DISTINCT _file), count() FROM file(currentDatabase() || '/05459_now_*.csv', 'CSV', 'n UInt64');
Loading